You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
136 lines
4.1 KiB
136 lines
4.1 KiB
import sqlite3
|
|
import csv
|
|
from typing import List, Dict, Optional
|
|
from bs4 import BeautifulSoup
|
|
import requests
|
|
import re
|
|
from fuzzywuzzy import fuzz
|
|
|
|
# Import necessary functions from DetermineOriginGame.py
|
|
from DetermineOriginGame import (
|
|
create_pokemon_index,
|
|
get_intro_generation,
|
|
get_locations_from_bulbapedia,
|
|
get_evolution_data_from_bulbapedia,
|
|
split_td_contents,
|
|
parse_form_information,
|
|
get_cached_data,
|
|
all_games,
|
|
pokemon_index,
|
|
cache,
|
|
read_pokemon_list
|
|
)
|
|
|
|
class Pokemon:
|
|
def __init__(self, number: int, name: str, form: Optional[str] = None):
|
|
self.number = number
|
|
self.name = name
|
|
self.form = form
|
|
self.introduced_in_gen: Optional[int] = None
|
|
self.encounters: Dict[str, List[str]] = {}
|
|
self.evolution_chain: List[Dict] = []
|
|
self.stage: Optional[str] = None
|
|
|
|
def create_database():
|
|
conn = sqlite3.connect('unprocessed_pokemon_database.db')
|
|
cursor = conn.cursor()
|
|
|
|
# Create tables
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS pokemon (
|
|
id INTEGER PRIMARY KEY,
|
|
national_dex_number INTEGER,
|
|
name TEXT,
|
|
form TEXT,
|
|
introduced_in_gen INTEGER
|
|
)
|
|
''')
|
|
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS encounters (
|
|
id INTEGER PRIMARY KEY,
|
|
pokemon_id INTEGER,
|
|
game TEXT,
|
|
location TEXT,
|
|
FOREIGN KEY (pokemon_id) REFERENCES pokemon (id)
|
|
)
|
|
''')
|
|
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS evolution_chain (
|
|
id INTEGER PRIMARY KEY,
|
|
pokemon_id INTEGER,
|
|
stage INTEGER,
|
|
evolves_from TEXT,
|
|
evolution_method TEXT,
|
|
FOREIGN KEY (pokemon_id) REFERENCES pokemon (id)
|
|
)
|
|
''')
|
|
|
|
conn.commit()
|
|
return conn
|
|
|
|
def extract_pokemon_data(pokemon_list: List[Pokemon], conn: sqlite3.Connection):
|
|
cursor = conn.cursor()
|
|
|
|
for pokemon in pokemon_list:
|
|
print(f"Processing {pokemon.name} ({pokemon.form})")
|
|
|
|
# Get introduction generation
|
|
pokemon.introduced_in_gen = get_intro_generation(pokemon.name, pokemon.form, cache)
|
|
|
|
# Get encounter data
|
|
encounter_data = get_locations_from_bulbapedia(pokemon.name, pokemon.form, cache)
|
|
for game, locations in encounter_data.items():
|
|
pokemon.encounters[game] = locations
|
|
|
|
# Get evolution data
|
|
pokemon.evolution_chain = get_evolution_data_from_bulbapedia(pokemon.name, pokemon.form, cache)
|
|
|
|
# Insert data into database
|
|
cursor.execute('''
|
|
INSERT INTO pokemon (national_dex_number, name, form, introduced_in_gen)
|
|
VALUES (?, ?, ?, ?)
|
|
''', (pokemon.number, pokemon.name, pokemon.form, pokemon.introduced_in_gen))
|
|
pokemon_id = cursor.lastrowid
|
|
|
|
for game, locations in pokemon.encounters.items():
|
|
for location in locations:
|
|
cursor.execute('''
|
|
INSERT INTO encounters (pokemon_id, game, location)
|
|
VALUES (?, ?, ?)
|
|
''', (pokemon_id, game, location))
|
|
|
|
if pokemon.evolution_chain:
|
|
for i, stage in enumerate(pokemon.evolution_chain):
|
|
previous_stage = None
|
|
if stage.previous_stage:
|
|
previous_stage = stage.previous_stage.pokemon
|
|
cursor.execute('''
|
|
INSERT INTO evolution_chain (pokemon_id, stage, evolves_from, evolution_method)
|
|
VALUES (?, ?, ?, ?)
|
|
''', (pokemon_id, i, previous_stage, stage.method))
|
|
|
|
conn.commit()
|
|
|
|
def read_and_convert_pokemon_list(filename: str) -> List[Pokemon]:
|
|
pokemon_list = read_pokemon_list(filename, 3000)
|
|
local_list = []
|
|
for entry in pokemon_list:
|
|
number = entry.number
|
|
name = entry.name
|
|
form = entry.form
|
|
local_list.append(Pokemon(number, name, form))
|
|
return local_list
|
|
|
|
def main():
|
|
get_cached_data()
|
|
conn = create_database()
|
|
pokemon_list = read_and_convert_pokemon_list('pokemon_home_list.csv')
|
|
create_pokemon_index(pokemon_list)
|
|
extract_pokemon_data(pokemon_list, conn)
|
|
conn.close()
|
|
print("Data extraction complete and stored in the database.")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|
|
|