diff --git a/DBEditor/DBEditor.py b/DBEditor/DBEditor.py index 81c413c..e208d14 100644 --- a/DBEditor/DBEditor.py +++ b/DBEditor/DBEditor.py @@ -20,8 +20,13 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from db_controller import DBController from pokemon_db_ui import PokemonUI from DataGatherers.update_location_information import process_pokemon_for_location_data +from DataGatherers.pokemondb_scraper import retrieve_all_pokemon_forms +from DataGatherers.update_storable_in_home import update_storable_in_home +from DataGatherers.update_location_information import profile_update_location_information +from DataGatherers.Update_evolution_information import update_evolution_chains from DataGatherers.cache_manager import CacheManager from event_system import event_system +from db_classes import PokemonForm import logging @@ -40,7 +45,6 @@ class PokemonFormGatherer(QThread): def run(self): cache = CacheManager() debugpy.debug_this_thread() - from DataGatherers.pokemondb_scraper import retrieve_all_pokemon_forms retrieve_all_pokemon_forms(cache, progress_callback=self.progress_signal.emit) self.finished_signal.emit() @@ -51,7 +55,6 @@ class PokemonHomeStatusUpdater(QThread): def run(self): cache = CacheManager() debugpy.debug_this_thread() - from DataGatherers.update_storable_in_home import update_storable_in_home update_storable_in_home(cache, progress_callback=self.progress_signal.emit) self.finished_signal.emit() @@ -61,10 +64,8 @@ class PokemonEvolutionUpdater(QThread): def run(self): time.sleep(0.1) - #pdb.set_trace() debugpy.debug_this_thread() cache = CacheManager() - from DataGatherers.Update_evolution_information import update_evolution_chains update_evolution_chains(cache, progress_callback=self.progress_signal.emit) self.finished_signal.emit() @@ -74,10 +75,9 @@ class PokemonEncounterUpdater(QThread): def run(self): time.sleep(0.1) - debugpy.debug_this_thread() + #debugpy.debug_this_thread() cache = CacheManager() - from DataGatherers.update_location_information import update_location_information - update_location_information(cache, progress_callback=self.progress_signal.emit) + profile_update_location_information(cache, progress_callback=self.progress_signal.emit) self.finished_signal.emit() class DBEditor(QMainWindow): @@ -443,9 +443,12 @@ class DBEditor(QMainWindow): generation_encounters = [] for encounter in encounters: game_generation = event_system.call_sync('get_game_generation', encounter[0]) + game_id = event_system.call_sync('get_game_id_for_name', encounter[0]) + encounter = encounter + (game_generation, game_id) if game_generation == target_generation: generation_encounters.append(encounter) if len(generation_encounters) > 0: + generation_encounters = sorted(generation_encounters, key=lambda x: x[12]) form_info = event_system.call_sync('get_pokemon_data', pfic) mark_id = event_system.call_sync('get_mark_for_game_name', generation_encounters[0][0]) if mark_id == None: @@ -463,14 +466,15 @@ class DBEditor(QMainWindow): pokemon_list = event_system.call_sync('get_pokemon_list') for pfic, name, form_name, national_dex in pokemon_list: - pokemon_data = event_system.call_sync('get_pokemon_data', pfic) + pokemon_data = event_system.call_sync('get_pokemon_form_by_pfic', pfic) - if pokemon_data[4] == False: + if pokemon_data.storable_in_home == False: continue; - self.logger.info(f"Determining mark for {name} {form_name if form_name else ''}") + self.logger.info(f"Determining mark for {pokemon_data.name} {pokemon_data.form_name if pokemon_data.form_name else ''}") + + target_generation = pokemon_data.generation - target_generation = pokemon_data[3] #Rule 1 # 1. If a pokemon form has a previous evolution from within the same generation, # use the mark of the previous evolution. This should be recursive within the same generation. @@ -484,9 +488,9 @@ class DBEditor(QMainWindow): if current_pfic == None: base_form_in_generation = last_pfic break - chain_pokemon_data = event_system.call_sync('get_pokemon_data', current_pfic[0]) - if chain_pokemon_data[3] == target_generation: - base_form_in_generation = current_pfic[0] + chain_pokemon_data = event_system.call_sync('get_pokemon_form_by_pfic', current_pfic[0]) + if chain_pokemon_data.generation == target_generation: + base_form_in_generation = current_pfic else: base_form_in_generation = last_pfic break @@ -503,7 +507,7 @@ class DBEditor(QMainWindow): if mark_id != None: event_system.emit_sync('assign_mark_to_form', (pfic, mark_id)) continue; - #Rule 2 + #Rule 2a # If a pokemon form has no previous evolution from within the same generation, # look at the encounters of the pokemon form from this generation and use the mark of the earliest # game you can encounter that form in from that generation @@ -511,55 +515,54 @@ class DBEditor(QMainWindow): if mark_id != None: event_system.emit_sync('assign_mark_to_form', (pfic, mark_id)) continue; - - - # this bit doesn't work as we exclude eveolve encounters from our dataset. - # this needs to change so we do include them, but filter them out of all our current queries - # then make a new API that includes them to solve this problem. - # We then find the earliest encounter in the for the base pokemon, that gets uus a game id - # we then use that game id to get the generation and run the determine_origin_mark using that generation - # for its evolution line. #Rule 3 # If there are no encounters for the pokemon form from this generation, # look to see if a previous evolution has an encounter from this generation, and use the mark of the earliest # game from this generation that the previous evolution is encounterable in. - if chain: - last_pfic = pfic - mark_identified = False - should_continue = True - while should_continue: - encounters = event_system.call_sync('get_encounter_locations', last_pfic) - if encounters: - earliest_game = 100 - for encounter in encounters: - game_id = event_system.call_sync('get_game_id_for_name', encounter[0]) - if game_id <= earliest_game: - earliest_game = game_id - if earliest_game < 100: - form_info = event_system.call_sync('get_pokemon_data', last_pfic) - mark_id = event_system.call_sync('get_mark_for_game', earliest_game) - if mark_id == None: - self.logger.info(f"No mark found for {form_info[0]} {form_info[1]}") - else: - mark_details = event_system.call_sync('get_mark_details', mark_id) - self.logger.info(f"Mark for {form_info[0]} {form_info[1]} is {mark_details[0]} - {mark_details[1]}") - event_system.emit_sync('assign_mark_to_form', (pfic, mark_id)) - should_continue = False - mark_identified = True + evolve_encounters = event_system.call_sync('get_evolve_encounters', pfic) + if evolve_encounters: + available_encounters = [] + for encounter in evolve_encounters: + game_id = encounter.game_id + game_info = event_system.call_sync('get_game_by_id', game_id) + if game_info[2] == target_generation: + available_encounters.append(encounter) + + if len(available_encounters) > 0: + available_encounters = sorted(available_encounters, key=lambda x: x.game_id) + mark_id = self.determine_origin_mark(available_encounters[0].from_pfic, target_generation) + if mark_id != None: + event_system.emit_sync('assign_mark_to_form', (pfic, mark_id)) + continue; + + encounters = event_system.call_sync('get_encounter_locations', pfic) + count = 0 + if encounters: + for encounter in encounters: + game_id = event_system.call_sync('get_game_id_for_name', encounter[0]) + game_generation = event_system.call_sync('get_game_generation', game_id) + if game_generation == target_generation: + count += 1 + if count == 0: + #Rule 2b + # Check to see if this is a sub-form pokemon, and if so, use the mark of the base form. + base = pfic[:-6] + alternate_forms = event_system.call_sync('find_default_form', base) + if alternate_forms: + form_found = False + for alternate_form in alternate_forms: + mark_id = self.determine_origin_mark(alternate_form.pfic, target_generation) + if mark_id != None: + event_system.emit_sync('assign_mark_to_form', (pfic, mark_id)) + form_found = True break; - - last_pfic = event_system.call_sync('get_evolution_parent', last_pfic) - if last_pfic == None: - should_continue = False - break - if mark_identified: - continue; + if form_found: + continue; #Rule 4 # If there are no encounters for the pokemon form or its evolution line from this generation, # use the mark of the earliest game of the generation is marked as being introducted in. - encounters = event_system.call_sync('get_encounter_locations', pfic) if encounters: earliest_game = 100 for encounter in encounters: @@ -577,6 +580,8 @@ class DBEditor(QMainWindow): event_system.emit_sync('assign_mark_to_form', (pfic, mark_id)) continue; + + def qt_message_handler(mode, context, message): print(f"Qt Message: {mode} {context} {message}") diff --git a/DBEditor/db_classes.py b/DBEditor/db_classes.py new file mode 100644 index 0000000..75029ba --- /dev/null +++ b/DBEditor/db_classes.py @@ -0,0 +1,18 @@ + +class PokemonForm: + def __init__(self, pfic: int, name: str, form_name: str, national_dex: int, generation: int, is_baby_form: bool, storable_in_home: bool): + self.pfic = pfic + self.name = name + self.form_name = form_name + self.national_dex = national_dex + self.generation = generation + self.is_baby_form = is_baby_form + self.storable_in_home = storable_in_home + +class EvolveEncounter: + def __init__(self, pfic: int, game_id: int, day: str, time: str, from_pfic: int): + self.pfic = pfic + self.game_id = game_id + self.day = day + self.time = time + self.from_pfic = from_pfic diff --git a/DBEditor/db_controller.py b/DBEditor/db_controller.py index 0bd980e..68f8bac 100644 --- a/DBEditor/db_controller.py +++ b/DBEditor/db_controller.py @@ -1,16 +1,30 @@ import sqlite3 import sys import os +import threading +from queue import Queue +from concurrent.futures import ThreadPoolExecutor from event_system import event_system +from db_classes import PokemonForm, EvolveEncounter def parse_pfic(pfic): parts = pfic.split('-') return tuple(int(part) if part.isdigit() else part for part in parts) class DBController: - def __init__(self): - self.conn = sqlite3.connect(':memory:', check_same_thread=False) # Use in-memory database for runtime + def __init__(self, db_path=':memory:', max_connections=10): + self.db_path = db_path + self.connection_pool = Queue(maxsize=max_connections) + self.lock = threading.Lock() + + for _ in range(max_connections): + conn = sqlite3.connect(db_path, check_same_thread=False) + self.connection_pool.put(conn) + + self.executor = ThreadPoolExecutor(max_workers=max_connections) + + self.conn = sqlite3.connect(db_path, check_same_thread=False) self.cursor = self.conn.cursor() event_system.add_listener('get_pokemon_data', self.load_pokemon_details) @@ -38,6 +52,10 @@ class DBController: event_system.add_listener('get_mark_for_game', self.get_mark_for_game) event_system.add_listener('get_mark_for_game_name', self.get_mark_for_game_name) event_system.add_listener('get_mark_details', self.get_mark_details) + event_system.add_listener('get_evolve_encounters', self.get_evolve_encounters) + event_system.add_listener('get_game_by_id', self.get_game_by_id) + event_system.add_listener('get_pokemon_form_by_pfic', self.get_pokemon_form_by_pfic) + event_system.add_listener('find_default_form', self.find_default_form) def init_database(self): disk_conn = sqlite3.connect('pokemon_forms.db') @@ -53,6 +71,7 @@ class DBController: self.create_encounters_table(disk_cursor) self.create_mark_table(disk_cursor) self.create_form_marks_table(disk_cursor) + self.create_evolve_encounter_table(disk_cursor) # Commit changes to the file-based database disk_conn.commit() @@ -60,6 +79,16 @@ class DBController: # Copy the file-based database to the in-memory database disk_conn.backup(self.conn) + temp_connections = [] + for _ in range(self.connection_pool.qsize()): + conn = self.connection_pool.get() + disk_conn.backup(conn) + temp_connections.append(conn) + + # Put all connections back into the pool + for conn in temp_connections: + self.connection_pool.put(conn) + # Close the file-based database connection disk_conn.close() @@ -168,6 +197,7 @@ class DBController: ("Sinnoh", "images/marks/BDSP_icon_HOME.png", ["Brilliant Diamond", "Shining Pearl"]), ("Hisui", "images/marks/Arceus_mark_HOME.png", ["Legends Arceus"]), ("Paldea", "images/marks/Paldea_icon_HOME.png", ["Scarlet", "Violet", "The Teal Mask", "The Hidden Treasure of Area Zero"]), + ("Markless", "images/marks/Markless_icon_HOME.png", ["Ruby", "Sapphire", "Emerald", "FireRed", "LeafGreen", "Diamond", "Pearl", "Platinum", "HeartGold", "SoulSilver", "Black", "White", "Black 2", "White 2"]), ] # Check if marks already exist @@ -301,6 +331,21 @@ class DBController: ) ''') + def create_evolve_encounter_table(self, cursor): + cursor.execute(''' + CREATE TABLE IF NOT EXISTS evolve_encounters ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + pfic TEXT, + game_id INTEGER, + day TEXT, + time TEXT, + from_pfic TEXT, + FOREIGN KEY (pfic) REFERENCES pokemon_forms (PFIC), + FOREIGN KEY (game_id) REFERENCES games (id), + FOREIGN KEY (from_pfic) REFERENCES pokemon_forms (PFIC) + ) + ''') + def load_pokemon_details(self, pfic): self.cursor.execute(''' SELECT pf.name, pf.form_name, pf.national_dex, pf.generation, ps.storable_in_home, pf.is_baby_form @@ -391,27 +436,21 @@ class DBController: result = self.cursor.fetchone() return result[0] if result else None - def get_game_id_for_name(self, game): - # First, try to find the game in the main games table - self.cursor.execute(''' - SELECT id - FROM games - WHERE name = ? - ''', (game,)) - result = self.cursor.fetchone() - + def get_game_id_for_name(self, game_name): + query = "SELECT id FROM games WHERE name = ?" + result = self.execute_query(query, (game_name,)) if result: - return result[0] + return result[0][0] - # If not found, check the alternate_game_names table - self.cursor.execute(''' - SELECT game_id - FROM alternate_game_names - WHERE alternate_name = ? - ''', (game,)) - result = self.cursor.fetchone() - - return result[0] if result else None + # If not found in main table, check alternate_game_names + query = """ + SELECT g.id + FROM games g + JOIN alternate_game_names agn ON g.id = agn.game_id + WHERE agn.alternate_name = ? + """ + result = self.execute_query(query, (game_name,)) + return result[0][0] if result else None def get_pokemon_with_encounters(self, data): self.cursor.execute(''' @@ -516,3 +555,76 @@ class DBController: self.cursor.execute('SELECT name, icon_path FROM marks WHERE id = ?', (mark_id,)) return self.cursor.fetchone() + def get_evolve_encounters(self, data): + pfic = data + self.cursor.execute('SELECT * FROM evolve_encounters WHERE pfic = ?', (pfic,)) + evolve_encounters = self.cursor.fetchall() + return [EvolveEncounter(encounter[1], encounter[2], encounter[3], encounter[4], encounter[5]) for encounter in evolve_encounters] + + def get_game_by_id(self, data): + game_id = data + self.cursor.execute('SELECT * FROM games WHERE id = ?', (game_id,)) + result = self.cursor.fetchone() + return result + + def get_pokemon_form_by_pfic(self, data) -> PokemonForm: + pfic = data + self.cursor.execute(''' + SELECT pf.name, pf.form_name, pf.national_dex, pf.generation, ps.storable_in_home, pf.is_baby_form + FROM pokemon_forms pf + LEFT JOIN pokemon_storage ps ON pf.PFIC = ps.PFIC + WHERE pf.PFIC = ? + ''', (pfic,)) + result = self.cursor.fetchone() + return PokemonForm(pfic, result[0], result[1], result[2], result[3], result[5], result[4]) + + def find_default_form(self, data): + search_pfic = data + self.cursor.execute(''' + SELECT pf.PFIC, pf.name, pf.form_name, pf.national_dex, pf.generation, ps.storable_in_home, pf.is_baby_form + FROM pokemon_forms pf + LEFT JOIN pokemon_storage ps ON pf.PFIC = ps.PFIC + WHERE pf.PFIC LIKE ? + ''', (search_pfic + "%",)) + results = self.cursor.fetchall() + return [PokemonForm(result[0], result[1], result[2], result[3], result[4], result[6], result[5]) for result in results] + + def get_connection(self): + return self.connection_pool.get() + + def release_connection(self, conn): + self.connection_pool.put(conn) + + def execute_query_with_commit(self, query, params=None): + conn = self.get_connection() + try: + with conn: + cursor = conn.cursor() + if params: + cursor.execute(query, params) + else: + cursor.execute(query) + conn.commit() + return cursor.fetchall() + finally: + self.release_connection(conn) + + def execute_query(self, query, params=None): + conn = self.get_connection() + try: + with conn: + cursor = conn.cursor() + if params: + cursor.execute(query, params) + else: + cursor.execute(query) + return cursor.fetchall() + finally: + self.release_connection(conn) + + def close(self): + while not self.connection_pool.empty(): + conn = self.connection_pool.get() + conn.close() + self.executor.shutdown() + diff --git a/DataGatherers/DetermineOriginGame.py b/DataGatherers/DetermineOriginGame.py index feab6aa..7c65583 100644 --- a/DataGatherers/DetermineOriginGame.py +++ b/DataGatherers/DetermineOriginGame.py @@ -19,6 +19,9 @@ import sys sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from DataGatherers.cache_manager import CacheManager +import concurrent.futures +from concurrent.futures import ThreadPoolExecutor, as_completed +from functools import lru_cache # List of all main series Pokémon games in chronological order, with special games first in each generation all_games = [ @@ -28,7 +31,7 @@ all_games = [ "Platinum", "HeartGold", "SoulSilver", "Diamond", "Pearl", "Black 2", "White 2", "Black", "White", "X", "Y", "Omega Ruby", "Alpha Sapphire", - "Ultra Sun", "Ultra Moon", "Sun", "Moon", + "Ultra Sun", "Ultra Moon", "Sun", "Moon", "Let's Go Pikachu", "Let's Go Eevee", "Sword", "Shield", "Expansion Pass", "Brilliant Diamond", "Shining Pearl", "Legends: Arceus", @@ -371,24 +374,19 @@ def get_pokemon_data_bulbapedia(pokemon_name, cache: CacheManager): def split_td_contents(td): groups = [] current_group = [] - for content in td.contents: - if isinstance(content, Tag) and (content.name == 'br' or content.name == 'p'): + if isinstance(content, NavigableString): + text = content.strip() + if text: + current_group.append(content) + elif content.name == 'br': if current_group: - groups.append(BeautifulSoup('', 'html.parser').new_tag('div')) - for item in current_group: - groups[-1].append(copy.copy(item)) - if content.name == 'p': - groups[-1].append(copy.copy(content)) + groups.append(''.join(str(item) for item in current_group)) current_group = [] else: current_group.append(content) - if current_group: - groups.append(BeautifulSoup('', 'html.parser').new_tag('div')) - for item in current_group: - groups[-1].append(copy.copy(item)) - + groups.append(''.join(str(item) for item in current_group)) return groups def parse_form_information(html_content): @@ -588,145 +586,137 @@ def compare_forms(a, b): return False -def get_locations_from_bulbapedia(pokemon_name, form, cache: CacheManager, default_forms = None): +@lru_cache(maxsize=100) +def get_parsed_pokemon_page(pokemon_name, cache): page_data = get_pokemon_data_bulbapedia(pokemon_name, cache) - if not page_data: - return None - - soup = BeautifulSoup(page_data, 'html.parser') + return BeautifulSoup(page_data, 'html.parser') if page_data else None - locations_section = soup.find('span', id='Game_locations') - if not locations_section: +def get_locations_from_bulbapedia(pokemon_name, form, cache: CacheManager, default_forms=None): + soup = get_parsed_pokemon_page(pokemon_name, cache) + if not soup: return None + + # Try different methods to find the locations table + locations_table = None + possible_headers = ['Game locations', 'In side games', 'In spin-off games'] - locations_table = locations_section.find_next('table', class_='roundy') + for header in possible_headers: + span = soup.find('span', id=header.replace(' ', '_')) + if span: + locations_table = span.find_next('table', class_='roundy') + if locations_table: + break + if not locations_table: + print(f"Warning: Couldn't find locations table for {pokemon_name}") return None - - raw_game_locations = {} - # Ok so the table is a bit of a mess. It has some nested tables and stuff. - # In each row is a nested table with all the games in a generation. - # Next is another nexted table, but i can't tell what for. - # within that nested table, is another nested table with the games, either the release pair or a single game spanning two columns. - # Next to that is another nested table with the locations. + raw_game_locations = {} - generation_tbody = locations_table.find('tbody', recursive=False) - generation_rows = generation_tbody.find_all('tr', recursive=False) - for generation_row in generation_rows: - random_nested_td = generation_row.find('td', recursive=False) - if not random_nested_td: - continue - random_nested_table = random_nested_td.find('table', recursive=False) - if not random_nested_table: + # Process game locations + for row in locations_table.select('tr'): + games = row.select('th') + locations = row.select('td') + + if len(games) != len(locations): continue - random_nested_tbody = random_nested_table.find('tbody', recursive=False) - random_nested_rows = random_nested_tbody.find_all('tr', recursive=False) - for nested_row in random_nested_rows: - if 'Generation' in nested_row.get_text(strip=True): - continue - - games_container_td = nested_row.find('td', recursive=False) - if not games_container_td: - continue - games_container_table = games_container_td.find('table', recursive=False) - if not games_container_table: - continue - games_container_tbody = games_container_table.find('tbody', recursive=False) - games_container_rows = games_container_tbody.find_all('tr', recursive=False) - for games_container_row in games_container_rows: - games = games_container_row.find_all('th') - for game in games: - raw_game = game.get_text(strip=True) - if raw_game not in all_games: - continue - locations_container_td = games_container_row.find('td', recursive=False) - if not locations_container_td: - continue - locations_container_table = locations_container_td.find('table', recursive=False) - if not locations_container_table: - continue - locations_container_tbody = locations_container_table.find('tbody', recursive=False) - locations = locations_container_tbody.find_all('td') - for location in locations: - groups = split_td_contents(location) - for group in groups: - if raw_game not in raw_game_locations: - raw_game_locations[raw_game] = [] - raw_game_locations[raw_game].append(group) + for game, location in zip(games, locations): + raw_game = game.get_text(strip=True) + if raw_game in all_games: + groups = split_td_contents(location) + raw_game_locations.setdefault(raw_game, []).extend(groups) + # Process events events_section = soup.find('span', id='In_events') - event_tables = {} - if events_section: - event_header = events_section.parent - - variant = "" - for sibling in event_header.find_next_siblings(): - if sibling.name == 'h4' or "held" in sibling.getText(strip=True).lower(): - break - if sibling.name == 'h5': - variant = sibling.get_text(strip=True) - if sibling.name == 'table': - event_tables[variant] = sibling + event_tables = process_event_tables(events_section) if events_section else {} - game_locations = {} - for raw_game, raw_locations in raw_game_locations.items(): - if form is None: - for raw_location in raw_locations: - raw_text = raw_location.get_text() - forms = parse_form_information(str(raw_location)) - if len(forms) > 0: - for form_info in forms: - main_form = form_info["main_form"] - - if default_forms and main_form and main_form in default_forms: - main_form = None + # Process game locations in parallel + with ThreadPoolExecutor() as executor: + futures = {executor.submit(process_game_locations, raw_game, raw_locations, form, default_forms): raw_game + for raw_game, raw_locations in raw_game_locations.items()} + + game_locations = {} + for future in as_completed(futures): + raw_game = futures[future] + result = future.result() + if result: + game_locations[raw_game] = result + + # Process event tables + for variant in event_tables: + if (variant == pokemon_name and form is None) or (form and form in variant): + process_event_table(event_tables[variant], game_locations) - if main_form and (main_form != "All Forms" and main_form != "Kantonian Form" and main_form != "All Sizes"): - continue + return game_locations - record_location_info(raw_game, game_locations, raw_location, raw_text) - else: - record_location_info(raw_game, game_locations, raw_location, raw_text) - else: - for raw_location in raw_locations: - forms = parse_form_information(str(raw_location)) +def process_event_tables(events_section): + event_tables = {} + if events_section: + next_element = events_section.find_next_sibling() + while next_element and next_element.name != 'h3': + if next_element.name == 'h4': + variant = next_element.text.strip() + table = next_element.find_next_sibling('table', class_='roundy') + if table: + event_tables[variant] = table + next_element = next_element.find_next_sibling() + return event_tables + +def process_event_table(table, game_locations): + for row in table.find_all('tr')[1:]: # Skip header row + cells = row.find_all('td') + if len(cells) >= 3: + game = cells[0].text.strip() + location = cells[2].text.strip() + if game in all_games: + if game not in game_locations: + game_locations[game] = [] + game_locations[game].append({"location": f"Event: {location}", "tag": str(cells[2])}) + +def process_game_locations(raw_game, raw_locations, form, default_forms): + locations = [] + + for raw_location in raw_locations: + raw_text = raw_location + forms = parse_form_information(raw_location) + if form is None: + if len(forms) > 0: for form_info in forms: main_form = form_info["main_form"] - sub_form = form_info["sub_form"] + if default_forms and main_form and main_form in default_forms: + main_form = None - if not main_form: + if main_form and (main_form != "All Forms" and main_form != "Kantonian Form" and main_form != "All Sizes"): continue - if main_form == "All Forms" or main_form == "Kantonian Form" or main_form == "All Sizes": - main_form = form + locations.append({"location": raw_text, "tag": raw_location}) + else: + locations.append({"location": raw_text, "tag": raw_location}) + elif len(forms) > 0: + for form_info in forms: + if form_matches(form_info, form, default_forms): + locations.append({"location": raw_text, "tag": raw_location}) + + return locations if locations else None - main_form_match = compare_forms(form, main_form) - if not main_form_match: - main_form_match = fuzz.partial_ratio(form.lower(), main_form.lower()) >= 80 +def form_matches(form_info, form, default_forms): + main_form = form_info["main_form"] + sub_form = form_info["sub_form"] - sub_form_match = compare_forms(form, sub_form) - if not sub_form_match: - sub_form_match = False if not sub_form else fuzz.partial_ratio(form.lower(), sub_form.lower()) >= 80 + if default_forms and main_form and main_form in default_forms: + main_form = None - if main_form_match or sub_form_match: - raw_text = raw_location.get_text() - record_location_info(raw_game, game_locations, raw_location, raw_text) + if main_form is None: + return False - # For Later - for variant in event_tables: - if (variant == pokemon_name and form is None)or (form and form in variant): - games_container_rows = event_tables[variant].find_all('tr') - for game_row in games_container_rows: - entries = game_row.find_all('td') - if len(entries) > 1: - games_string = entries[0].find('a').get('title') - for game in all_games: - if game in games_string: - record_location_info(game, game_locations, "Event", "Event") + if main_form in ["All Forms", "Kantonian Form", "All Sizes"]: + return True - return game_locations + main_form_match = compare_forms(form, main_form) or fuzz.partial_ratio(form.lower(), main_form.lower()) >= 80 + sub_form_match = compare_forms(form, sub_form) or (sub_form and fuzz.partial_ratio(form.lower(), sub_form.lower()) >= 80) + + return main_form_match or sub_form_match def record_location_info(raw_game, game_locations, raw_location, raw_text): if raw_game not in game_locations: @@ -916,3 +906,4 @@ if __name__ == "__main__": cache.close() print(f"Earliest obtainable games and encounter locations determined for {len(pokemon_list)} Pokémon and saved to pokemon_earliest_games.csv") + diff --git a/DataGatherers/Update_evolution_information.py b/DataGatherers/Update_evolution_information.py index c8838d7..d6ea4d0 100644 --- a/DataGatherers/Update_evolution_information.py +++ b/DataGatherers/Update_evolution_information.py @@ -13,6 +13,8 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from DataGatherers.cache_manager import CacheManager from DataGatherers.DetermineOriginGame import get_evolution_data_from_bulbapedia +from db_controller import DBController + logger = logging.getLogger('ui_feedback') @dataclass @@ -37,14 +39,29 @@ def create_evolution_table(): conn.commit() return conn -def insert_evolution_info(conn, evolution_info: EvolutionInfo): - cursor = conn.cursor() - cursor.execute(''' +def sanitize_for_logging(text: str) -> str: + # Replace problematic Unicode characters with ASCII equivalents + replacements = { + '→': '->', + '←': '<-', + '⇒': '=>', + '⇐': '<=', + '↗': '-^>', + '↖': '<^-', + '♀': 'f', + '♂': 'm' + } + for unicode_char, ascii_char in replacements.items(): + text = text.replace(unicode_char, ascii_char) + return text + +def insert_evolution_info(db_controller, evolution_info: EvolutionInfo): + db_controller.execute_query_with_commit(''' INSERT OR REPLACE INTO evolution_chains (from_pfic, to_pfic, method) VALUES (?, ?, ?) ''', (evolution_info.from_pfic, evolution_info.to_pfic, evolution_info.method)) - conn.commit() + logger.info(sanitize_for_logging(f"Adding a link from {evolution_info.from_pfic} to {evolution_info.to_pfic}, via {evolution_info.method}")) def strip_pokemon_name(pokemon_name: str, form_name: str) -> str: """Remove the Pokémon's name from the form name if present.""" @@ -61,10 +78,9 @@ def fuzzy_match_form(form1: str, form2: str, threshold: int = 80) -> bool: return form1 == form2 return fuzz.ratio(form1.lower(), form2.lower()) >= threshold -def get_pokemon_form_by_name(conn, name: str, form: Optional[str] = None, threshold: int = 80, gender: Optional[str] = None) -> Optional[str]: - cursor = conn.cursor() - cursor.execute('SELECT PFIC, name, form_name FROM pokemon_forms WHERE name = ?', (name,)) - results = cursor.fetchall() +def get_pokemon_form_by_name(db_controller, name: str, form: Optional[str] = None, threshold: int = 80, gender: Optional[str] = None) -> Optional[str]: + + results = db_controller.execute_query('SELECT PFIC, name, form_name FROM pokemon_forms WHERE name = ?', (name,)) if not results: return None @@ -74,12 +90,12 @@ def get_pokemon_form_by_name(conn, name: str, form: Optional[str] = None, thresh if results[0][2] == None: return results[0][0] else: - return get_pokemon_form_by_name(conn, name, "Male", threshold=100, gender=gender) + return get_pokemon_form_by_name(db_controller, name, "Male", threshold=100, gender=gender) else: return results[0][0] # Return the PFIC of the first result if no form is specified if gender: - gendered_form = get_pokemon_form_by_name(conn, name, gender, threshold=100) + gendered_form = get_pokemon_form_by_name(db_controller, name, gender, threshold=100) if gendered_form: return gendered_form @@ -96,68 +112,76 @@ def get_pokemon_form_by_name(conn, name: str, form: Optional[str] = None, thresh return None -def process_evolution_chain(conn, evolution_chain, cache, gender: Optional[str] = None): +def process_evolution_chain(db_controller, evolution_chain, cache, gender: Optional[str] = None): for stage in evolution_chain: - from_pfic = get_pokemon_form_by_name(conn, stage.pokemon, stage.form, gender=gender) + from_pfic = get_pokemon_form_by_name(db_controller, stage.pokemon, stage.form, gender=gender) if not from_pfic: logger.warning(f"Could not find PFIC for {stage.pokemon} {stage.form}") continue if stage.next_stage: - to_pfic = get_pokemon_form_by_name(conn, stage.next_stage.pokemon, stage.next_stage.form, gender=gender) + to_pfic = get_pokemon_form_by_name(db_controller, stage.next_stage.pokemon, stage.next_stage.form, gender=gender) if to_pfic: evolution_info = EvolutionInfo(from_pfic, to_pfic, stage.next_stage.method) - insert_evolution_info(conn, evolution_info) + insert_evolution_info(db_controller, evolution_info) if "breed" in stage.next_stage.method.lower(): - update_pokemon_baby_status(conn, from_pfic, True) + update_pokemon_baby_status(db_controller, from_pfic, True) for branch in stage.branches: - to_pfic = get_pokemon_form_by_name(conn, branch.pokemon, branch.form, gender=gender) + to_pfic = get_pokemon_form_by_name(db_controller, branch.pokemon, branch.form, gender=gender) if to_pfic: evolution_info = EvolutionInfo(from_pfic, to_pfic, branch.method) - insert_evolution_info(conn, evolution_info) + insert_evolution_info(db_controller, evolution_info) if "breed" in branch.method.lower(): - update_pokemon_baby_status(conn, from_pfic, True) + update_pokemon_baby_status(db_controller, from_pfic, True) -def update_pokemon_baby_status(conn, from_pfic, is_baby_form): - cursor = conn.cursor() - cursor.execute(''' +def update_pokemon_baby_status(db_controller, from_pfic, is_baby_form): + db_controller.execute_query_with_commit(''' UPDATE pokemon_forms SET is_baby_form = ? WHERE PFIC = ? ''', (is_baby_form, from_pfic)) - conn.commit() -def update_evolution_chains(cache, progress_callback=None): - conn = create_evolution_table() - cursor = conn.cursor() - cursor.execute('SELECT DISTINCT name, form_name FROM pokemon_forms') - pokemon_forms = cursor.fetchall() - - for name, form in pokemon_forms: - if progress_callback: - progress_callback(f"Processing {name} {form if form else ''}") - - if form and name in form: - form = form.replace(name, "").strip() - - gender = None - if form and "male" in form.lower(): - gender = form - form = None - - evolution_chain = get_evolution_data_from_bulbapedia(name, form, cache, gender) - if evolution_chain: - if name == "Tauros": # Bulbapedia has a weird formatting for Tauros. - for stage in evolution_chain: - if stage.form: - stage.form = stage.form.replace("Paldean Form(", "").replace(")", "").strip() - process_evolution_chain(conn, evolution_chain, cache, gender) - - conn.close() +def update_evolution_chains(cache, progress_callback=None): + db_controller = DBController('pokemon_forms.db', max_connections=20) + try: + db_controller.execute_query_with_commit('BEGIN') + pokemon_forms = db_controller.execute_query(''' + SELECT pf.name, pf.form_name + FROM pokemon_forms pf + ORDER BY pf.national_dex, pf.form_name + ''') + + for name, form in pokemon_forms: + if progress_callback: + progress_callback(f"Processing {name} {form if form else ''}") + + if form and name in form: + form = form.replace(name, "").strip() + + gender = None + if form and "male" in form.lower(): + gender = form + form = None + + evolution_chain = get_evolution_data_from_bulbapedia(name, form, cache, gender) + if evolution_chain: + if name == "Tauros": # Bulbapedia has a weird formatting for Tauros. + for stage in evolution_chain: + if stage.form: + stage.form = stage.form.replace("Paldean Form(", "").replace(")", "").strip() + process_evolution_chain(db_controller, evolution_chain, cache, gender) + db_controller.execute_query_with_commit('COMMIT') + except Exception as e: + # Rollback in case of error + db_controller.execute_query_with_commit('ROLLBACK') + logger.error(f"Error updating evolution chains: {str(e)}") + raise + finally: + db_controller.close() if __name__ == "__main__": cache = CacheManager() diff --git a/DataGatherers/cache_manager.py b/DataGatherers/cache_manager.py index 29a233d..731aaac 100644 --- a/DataGatherers/cache_manager.py +++ b/DataGatherers/cache_manager.py @@ -2,40 +2,102 @@ import sqlite3 import json import time import requests -from typing import Any, Optional +from typing import Any, Optional, Dict, List +import pickle +from concurrent.futures import ThreadPoolExecutor +from threading import Lock class CacheManager: - def __init__(self, db_name: str = 'pokemon_cache.db'): - self.conn = sqlite3.connect(db_name) - self.cursor = self.conn.cursor() + def __init__(self, db_name: str = 'pokemon_cache.db', max_connections: int = 10): + self.db_name = db_name + self.memory_cache: Dict[str, Any] = {} + self.expiry_disabled = True + self.lock = Lock() + self.connection_pool = ThreadPoolExecutor(max_workers=max_connections) self._create_cache_table() + self.fetch_url_lock = Lock() + + def _create_connection(self): + return sqlite3.connect(self.db_name) def _create_cache_table(self): - self.cursor.execute(''' - CREATE TABLE IF NOT EXISTS cache ( - key TEXT PRIMARY KEY, - value TEXT, - timestamp FLOAT - ) - ''') - self.conn.commit() + with self._create_connection() as conn: + cursor = conn.cursor() + cursor.execute(''' + CREATE TABLE IF NOT EXISTS cache ( + key TEXT PRIMARY KEY, + value BLOB, + timestamp FLOAT + ) + ''') + conn.commit() def get(self, key: str) -> Optional[Any]: - self.cursor.execute('SELECT value, timestamp FROM cache WHERE key = ?', (key,)) - result = self.cursor.fetchone() + # Check memory cache first + with self.lock: + if key in self.memory_cache: + return self.memory_cache[key] + + # If not in memory, check database + def db_get(): + with self._create_connection() as conn: + cursor = conn.cursor() + cursor.execute('SELECT value, timestamp FROM cache WHERE key = ?', (key,)) + result = cursor.fetchone() + if result: + value, timestamp = result + return pickle.loads(value) + return None + + result = self.connection_pool.submit(db_get).result() if result: - value, timestamp = result - return json.loads(value) - return None + with self.lock: + self.memory_cache[key] = result + return result def set(self, key: str, value: Any): - serialized_value = json.dumps(value) + serialized_value = pickle.dumps(value) timestamp = time.time() - self.cursor.execute(''' - INSERT OR REPLACE INTO cache (key, value, timestamp) - VALUES (?, ?, ?) - ''', (key, serialized_value, timestamp)) - self.conn.commit() + + with self.lock: + self.memory_cache[key] = value + + def db_set(): + with self._create_connection() as conn: + cursor = conn.cursor() + cursor.execute(''' + INSERT OR REPLACE INTO cache (key, value, timestamp) + VALUES (?, ?, ?) + ''', (key, serialized_value, timestamp)) + conn.commit() + + self.connection_pool.submit(db_set) + + def bulk_get(self, keys: List[str]) -> Dict[str, Any]: + results = {} + db_keys = [] + + with self.lock: + for key in keys: + if key in self.memory_cache: + results[key] = self.memory_cache[key] + else: + db_keys.append(key) + + if db_keys: + def db_bulk_get(): + with self._create_connection() as conn: + cursor = conn.cursor() + placeholders = ','.join('?' for _ in db_keys) + cursor.execute(f'SELECT key, value FROM cache WHERE key IN ({placeholders})', db_keys) + return {key: pickle.loads(value) for key, value in cursor.fetchall()} + + db_results = self.connection_pool.submit(db_bulk_get).result() + with self.lock: + self.memory_cache.update(db_results) + results.update(db_results) + + return results def fetch_url(self, url: str, force_refresh: bool = False, expiry: int = 86400) -> Optional[str]: cache_key = f"url_{url}" @@ -43,15 +105,16 @@ class CacheManager: cached_data = self.get(cache_key) if cached_data: cached_time = cached_data['timestamp'] - if time.time() - cached_time < expiry: + if time.time() - cached_time < expiry or self.expiry_disabled: return cached_data['content'] + #with self.fetch_url_lock: print(f"Fetching URL: {url}") response = requests.get(url) if response.status_code == 200: content = response.text self.set(cache_key, { - 'content': content, + 'content': content, 'timestamp': time.time() }) time.sleep(1) @@ -59,7 +122,7 @@ class CacheManager: return None def close(self): - self.conn.close() + self.connection_pool.shutdown() # Usage example if __name__ == "__main__": diff --git a/DataGatherers/pokemondb_scraper.py b/DataGatherers/pokemondb_scraper.py index f946f21..36f3e68 100644 --- a/DataGatherers/pokemondb_scraper.py +++ b/DataGatherers/pokemondb_scraper.py @@ -6,6 +6,8 @@ import os import sqlite3 import sys import logging +import re +import unicodedata sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -122,6 +124,32 @@ def get_pokemon_sprites_page_data(cache: CacheManager, pokemon_name: str): url = f"https://pokemondb.net/sprites/{pokemon_name}" return cache.fetch_url(url) +def get_pokemon_dex_page(cache: CacheManager, pokemon_name: str): + url = f"https://pokemondb.net/pokedex/{pokemon_name}" + return cache.fetch_url(url) + +def remove_accents(input_str): + nfkd_form = unicodedata.normalize('NFKD', input_str) + return u"".join([c for c in nfkd_form if not unicodedata.combining(c)]) + +def compare_forms(a, b): + if a == None or b == None: + return False + + if a == b: + return True + + temp_a = a.lower().replace("forme", "").replace("form", "").replace("é", "e").strip() + temp_b = b.lower().replace("forme", "").replace("form", "").replace("é", "e").strip() + + temp_a = temp_a.replace("deputante", "debutante").replace("p'au", "pa'u").replace("blood moon", "bloodmoon") + temp_b = temp_b.replace("deputante", "debutante").replace("p'au", "pa'u").replace("blood moon", "bloodmoon") + + if temp_a == temp_b: + return True + + return False + def download_image(url, filename): response = requests.get(url) if response.status_code == 200: @@ -205,18 +233,73 @@ def retrieve_all_pokemon_forms(cache: CacheManager, progress_callback=None): form_name = "None" if sprite.find('small'): - form_name = sprite.find('small').get_text(strip=True) + smalls = sprite.find_all('small') + form_name = "" + for small in smalls: + form_name += small.get_text(strip=True) + " " + form_name = form_name.strip() logger.info(f'{sprite_url}, {form_name}') if form_name != "None": form += 1 gender = 0 - if form_name == "Female": + if form_name.startswith("Male"): form -= 1 gender = 1 - elif form_name == "Male": + elif form_name.startswith("Female"): form -= 1 gender = 2 + dex_page_data = get_pokemon_dex_page(cache, pokemon_name.replace("'", "").replace(".", "-").replace(" ", "")) + if dex_page_data: + dex_soup = BeautifulSoup(dex_page_data, 'html.parser') + + #Find a heading that has the pokemon name in it + dex_header = dex_soup.find('h1', string=pokemon_name) + if dex_header: + #The next

tag contains the generation number, in the format "{pokemon name} is a {type}(/{2nd_type}) type Pokémon introduced in Generation {generation number}." + generation_tag = dex_header.find_next('p') + dex_text = generation_tag.get_text() + pattern = r'^(.+?) is a (\w+)(?:/(\w+))? type Pokémon introduced in Generation (\d+)\.$' + match = re.match(pattern, dex_text) + if match: + name, type1, type2, gen = match.groups() + generation = int(gen) + + if form_name != "None": + next_tag = generation_tag.find_next('p') + if next_tag: + extra_text = next_tag.get_text() + extra_text = remove_accents(extra_text) + form_pattern = r'a(?:n)? (\w+) Form(?:,)? introduced in (?:the )?([\w\s:]+)(?:\/([\w\s:]+))?' + update_pattern = r'a(?:n)? (\w+) form(?:,)? available in the latest update to ([\w\s:]+)(?:& ([\w\s:]+))?' + multiple_forms_pattern = r'has (?:\w+) new (\w+) Form(?:s)?(?:,)? available in (?:the )?([\w\s:]+)(?:& ([\w\s:]+))?' + expansion_pass_pattern = r'a(?:n)? (\w+) form(?:,)? introduced in the Crown Tundra Expansion Pass to ([\w\s:]+)(?:& ([\w\s:]+))?' + patterns = [form_pattern, update_pattern, multiple_forms_pattern, expansion_pass_pattern] + test_form = form_name.replace(pokemon_name, "").replace("Male", "").replace("Female", "").strip() + if pokemon_name == "Tauros" and (form_name == "Aqua Breed" or form_name == "Blaze Breed" or form_name == "Combat Breed"): + test_form = "Paldean" + for pattern in patterns: + matches = re.findall(pattern, extra_text, re.IGNORECASE) + generation_found = False + for i, (regional, game1, game2) in enumerate(matches, 1): + if compare_forms(test_form, regional): + target_game = game1.replace("Pokemon", "").strip() + cursor = pokemon_db_conn.cursor() + cursor.execute(''' + SELECT g.generation + FROM games g + LEFT JOIN alternate_game_names agn ON g.id = agn.game_id + WHERE g.name = ? OR agn.alternate_name = ? + LIMIT 1 + ''', (target_game, target_game)) + result = cursor.fetchone() + if result: + generation = result[0] + generation_found = True + break + if generation_found: + break + pokemon_form = PokemonForm( id=format_pokemon_id(national_dex_index, generation, form, gender), name=pokemon_name, diff --git a/DataGatherers/update_location_information.py b/DataGatherers/update_location_information.py index 9f3de79..cd6ae1b 100644 --- a/DataGatherers/update_location_information.py +++ b/DataGatherers/update_location_information.py @@ -12,10 +12,14 @@ from bs4 import BeautifulSoup, Tag import re import time import unicodedata +import concurrent.futures +from concurrent.futures import ThreadPoolExecutor, as_completed import logging logger = logging.getLogger('ui_feedback') +from db_controller import DBController # Import your DBController + def create_encounters_table(): conn = sqlite3.connect('pokemon_forms.db') cursor = conn.cursor() @@ -227,9 +231,7 @@ def build_query_string(table_name, criteria): return query + " AND ".join(conditions), values -def save_encounter(conn, pfic, game, location, days, times, dual_slot, static_encounter, static_encounter_count, extra_text, stars, rods, fishing, starter): - cursor = conn.cursor() - +def save_encounter(db_controller, pfic, game, location, days, times, dual_slot, static_encounter, static_encounter_count, extra_text, stars, rods, fishing, starter): extra_text_str = ' '.join(extra_text) if extra_text else None stars_str = ','.join(sorted(stars)) if stars else None rods_str = ','.join(sorted(rods)) if rods else None @@ -264,29 +266,25 @@ def save_encounter(conn, pfic, game, location, days, times, dual_slot, static_en criteria["time"] = None query, values = build_query_string("encounters", criteria) # Check if an identical record already exists - cursor.execute(query, values) - - encounter = cursor.fetchone() + result = db_controller.execute_query(query, values) - if encounter == None or encounter[0] == 0: - cursor.execute(insert_query, (pfic, game_id, location, day, None, dual_slot, static_encounter_count, + if not result: + db_controller.execute_query_with_commit(insert_query, (pfic, game_id, location, day, None, dual_slot, static_encounter_count, static_encounter, extra_text_str, stars_str, rods_str, fishing, starter)) logger.info(f"New encounter added for {pfic} in {game} at {location} on {day}") else: logger.info(f"Identical encounter already exists for {pfic} in {game} at {location} on {day}") - + elif len(times) > 0: for time in times: criteria["day"] = None criteria["time"] = time query, values = build_query_string("encounters", criteria) # Check if an identical record already exists - cursor.execute(query, values) - - encounter = cursor.fetchone() + result = db_controller.execute_query(query, values) - if encounter == None or encounter[0] == 0: - cursor.execute(insert_query, (pfic, game_id, location, None, time, dual_slot, static_encounter_count, + if not result: + db_controller.execute_query_with_commit(insert_query, (pfic, game_id, location, None, time, dual_slot, static_encounter_count, static_encounter, extra_text_str, stars_str, rods_str, fishing, starter)) logger.info(f"New encounter added for {pfic} in {game} at {location} at {time}") else: @@ -297,18 +295,71 @@ def save_encounter(conn, pfic, game, location, days, times, dual_slot, static_en criteria["time"] = None query, values = build_query_string("encounters", criteria) # Check if an identical record already exists - cursor.execute(query, values) - - encounter = cursor.fetchone() + result = db_controller.execute_query(query, values) - if encounter == None or encounter[0] == 0: - cursor.execute(insert_query, (pfic, game_id, location, None, None, dual_slot, static_encounter_count, + if not result: + db_controller.execute_query_with_commit(insert_query, (pfic, game_id, location, None, None, dual_slot, static_encounter_count, static_encounter, extra_text_str, stars_str, rods_str, fishing, starter)) logger.info(f"New encounter added for {pfic} in {game} at {location}") else: logger.info(f"Identical encounter already exists for {pfic} in {game} at {location}") - conn.commit() +def save_evolve_encounter(db_controller, pfic, game, days, times, from_pfic): + game_id = event_system.call_sync('get_game_id_for_name', game) + + insert_query = ''' + INSERT INTO evolve_encounters + (pfic, game_id, day, time, from_pfic) + VALUES (?, ?, ?, ?, ?) + ''' + + criteria = { + "pfic": pfic, + "game_id": game_id, + "day": None, + "time": None, + "from_pfic": from_pfic + } + + if len(days) > 0: + for day in days: + criteria["day"] = day + criteria["time"] = None + query, values = build_query_string("evolve_encounters", criteria) + # Check if an identical record already exists + result = db_controller.execute_query(query, values) + + if not result: + db_controller.execute_query_with_commit(insert_query, (pfic, game_id, day, None, from_pfic)) + logger.info(f"New evolve encounter added for {pfic} in {game} on {day}") + else: + logger.info(f"Identical evolve encounter already exists for {pfic} in {game} on {day}") + + elif len(times) > 0: + for time in times: + criteria["day"] = None + criteria["time"] = time + query, values = build_query_string("evolve_encounters", criteria) + # Check if an identical record already exists + result = db_controller.execute_query(query, values) + + if not result: + db_controller.execute_query_with_commit(insert_query, (pfic, game_id, None, time, from_pfic)) + logger.info(f"New evolve encounter added for {pfic} in {game} at {time}") + else: + logger.info(f"Identical evolve encounter already exists for {pfic} in {game} at {time}") + else: + criteria["day"] = None + criteria["time"] = None + query, values = build_query_string("evolve_encounters", criteria) + # Check if an identical record already exists + result = db_controller.execute_query(query, values) + + if not result: + db_controller.execute_query_with_commit(insert_query, (pfic, game_id, None, None, from_pfic)) + logger.info(f"New evolve encounter added for {pfic} in {game}") + else: + logger.info(f"Identical evolve encounter already exists for {pfic} in {game}") def compare_forms(a, b): if a == b: @@ -325,7 +376,7 @@ def compare_forms(a, b): return False -def process_pokemon_for_location_data(pfic, name, form, national_dex, default_forms, cache, conn): +def process_pokemon_for_location_data(pfic, name, form, national_dex, default_forms, cache, db_controller): logger.info(f"Processing {name} {form if form else ''}") if form and name in form: @@ -349,7 +400,13 @@ def process_pokemon_for_location_data(pfic, name, form, national_dex, default_fo if name.lower() == "ho-oh": name = "Ho-Oh" - if form and form.lower() == "female": + if form and form.startswith("Female"): + form = form.replace("Female", "").strip() + + if form and form.startswith("Male"): + form = form.replace("Male", "").strip() + + if form == "": form = None search_form = form @@ -358,7 +415,6 @@ def process_pokemon_for_location_data(pfic, name, form, national_dex, default_fo "trade", "time capsule", "unobtainable", - "evolve", "tradeversion", "poké transfer", "friend safari", @@ -402,36 +458,76 @@ def process_pokemon_for_location_data(pfic, name, form, national_dex, default_fo logger.info(f"Found in {encounter}:") print_encounter = False - remaining, details = extract_additional_information(location["tag"]) - routes, remaining = extract_routes(remaining) - logger.info(f"Routes: {routes}") - logger.info(f"Remaining: {remaining.strip()}") - logger.info(f"Details: {details}") + if "evolve" in test_location: + remaining, details = extract_additional_information(location["tag"]) + evolve_info = extract_evolve_information(remaining, db_controller) - if len(details["times"]) > 0: - logger.info("Stupid Data") + if evolve_info: + logger.info(f"Evolve Info: {evolve_info}") + save_evolve_encounter(db_controller, pfic, encounter, details["days"], details["times"], evolve_info["evolve_from"]) + else: + remaining, details = extract_additional_information(location["tag"]) + routes, remaining = extract_routes(remaining) + logger.info(f"Routes: {routes}") + logger.info(f"Remaining: {remaining.strip()}") + logger.info(f"Details: {details}") - for route in routes: - route_name = f"Route {route}" - save_encounter(conn, pfic, encounter, route_name, details["days"], details["times"], details["dual_slot"], details["static_encounter"], details["static_encounter_count"], details["extra_text"], details["stars"], details["Rods"], details["Fishing"], details["starter"] ) + if len(details["times"]) > 0: + logger.info("Stupid Data") - if remaining != "": - remaining_locations = remaining.replace(" and ", ",").split(",") - for remaining_location in remaining_locations: - if remaining_location.strip() == "": - continue + for route in routes: + route_name = f"Route {route}" + save_encounter(db_controller, pfic, encounter, route_name, details["days"], details["times"], details["dual_slot"], details["static_encounter"], details["static_encounter_count"], details["extra_text"], details["stars"], details["Rods"], details["Fishing"], details["starter"] ) - save_encounter(conn, pfic, encounter, remaining_location.strip(), details["days"], details["times"], details["dual_slot"], details["static_encounter"], details["static_encounter_count"], details["extra_text"], details["stars"], details["Rods"], details["Fishing"], details["starter"] ) + if remaining != "": + remaining_locations = remaining.replace(" and ", ",").split(",") + for remaining_location in remaining_locations: + if remaining_location.strip() == "": + continue + + save_encounter(db_controller, pfic, encounter, remaining_location.strip(), details["days"], details["times"], details["dual_slot"], details["static_encounter"], details["static_encounter_count"], details["extra_text"], details["stars"], details["Rods"], details["Fishing"], details["starter"] ) + +def extract_evolve_information(s: str, db_controller): + details = {} + if s is None or s == "": + return details + + s = s.replace("Evolve", "") + + parts = s.split(" ") + + if len(parts) >= 1: + target_pokemon = parts[0].strip() + + form = None + if "♀" in target_pokemon: + target_pokemon = target_pokemon.replace("♀", "").strip() + form = "Female" + + if "♂" in target_pokemon: + target_pokemon = target_pokemon.replace("♂", "").strip() + form = None + + result = db_controller.execute_query(''' + SELECT e.from_pfic + FROM evolution_chains e + JOIN pokemon_forms pf ON pf.PFIC = e.from_pfic + WHERE pf.name = ? + ''', (target_pokemon,)) + + if result: + details["evolve_from"] = result[0][0] + + return details def update_location_information(cache, progress_callback=None): - conn = create_encounters_table() - cursor = conn.cursor() - cursor.execute(''' - SELECT pf.PFIC, pf.name, pf.form_name, pf.national_dex - FROM pokemon_forms pf - ORDER BY pf.national_dex, pf.form_name - ''') - pokemon_forms = cursor.fetchall() + db_controller = DBController('pokemon_forms.db', max_connections=20) # Adjust max_connections as needed + + pokemon_forms = db_controller.execute_query(''' + SELECT pf.PFIC, pf.name, pf.form_name, pf.national_dex + FROM pokemon_forms pf + ORDER BY pf.national_dex, pf.form_name + ''') try: with open('./DataGatherers/DefaultForms.json', 'r') as f: @@ -439,14 +535,27 @@ def update_location_information(cache, progress_callback=None): except FileNotFoundError: default_forms = [] - for pfic, name, form, national_dex in pokemon_forms: - if progress_callback: - progress_callback(f"Processing {name} {form if form else ''}") - process_pokemon_for_location_data(pfic, name, form, national_dex, default_forms, cache, conn) + def process_single_pokemon(args): + pfic, name, form, national_dex = args + try: + process_pokemon_for_location_data(pfic, name, form, national_dex, default_forms, cache, db_controller) + if progress_callback: + progress_callback(f"Processed {name} {form if form else ''}") + except Exception as exc: + logger.error(f'Error processing {name} {form}: {exc}') + + with ThreadPoolExecutor(max_workers=10) as executor: + futures = [executor.submit(process_single_pokemon, form_data) for form_data in pokemon_forms] + + for future in as_completed(futures): + try: + future.result() + except Exception as exc: + logger.error(f'A task generated an exception: {exc}') - conn.close() + db_controller.close() -if __name__ == "__main__": - cache = CacheManager() - update_location_information(cache) - +import cProfile + +def profile_update_location_information(cache, progress_callback=None): + cProfile.runctx('update_location_information(cache, progress_callback)', globals(), locals(), 'profile_stats') diff --git a/Site/static/images/marks/Markless_icon_HOME.png b/Site/static/images/marks/Markless_icon_HOME.png new file mode 100644 index 0000000..ff3dc83 Binary files /dev/null and b/Site/static/images/marks/Markless_icon_HOME.png differ diff --git a/Site/static/images/pokemon/0019-07-001-0.png b/Site/static/images/pokemon/0019-07-001-0.png new file mode 100644 index 0000000..1295b7c Binary files /dev/null and b/Site/static/images/pokemon/0019-07-001-0.png differ diff --git a/Site/static/images/pokemon/0020-07-001-0.png b/Site/static/images/pokemon/0020-07-001-0.png new file mode 100644 index 0000000..1a246a0 Binary files /dev/null and b/Site/static/images/pokemon/0020-07-001-0.png differ diff --git a/Site/static/images/pokemon/0026-07-001-0.png b/Site/static/images/pokemon/0026-07-001-0.png new file mode 100644 index 0000000..2c40ef3 Binary files /dev/null and b/Site/static/images/pokemon/0026-07-001-0.png differ diff --git a/Site/static/images/pokemon/0027-07-001-0.png b/Site/static/images/pokemon/0027-07-001-0.png new file mode 100644 index 0000000..91cf7c4 Binary files /dev/null and b/Site/static/images/pokemon/0027-07-001-0.png differ diff --git a/Site/static/images/pokemon/0028-07-001-0.png b/Site/static/images/pokemon/0028-07-001-0.png new file mode 100644 index 0000000..e9de252 Binary files /dev/null and b/Site/static/images/pokemon/0028-07-001-0.png differ diff --git a/Site/static/images/pokemon/0037-07-001-0.png b/Site/static/images/pokemon/0037-07-001-0.png new file mode 100644 index 0000000..e48335c Binary files /dev/null and b/Site/static/images/pokemon/0037-07-001-0.png differ diff --git a/Site/static/images/pokemon/0038-07-001-0.png b/Site/static/images/pokemon/0038-07-001-0.png new file mode 100644 index 0000000..00e8428 Binary files /dev/null and b/Site/static/images/pokemon/0038-07-001-0.png differ diff --git a/Site/static/images/pokemon/0050-07-001-0.png b/Site/static/images/pokemon/0050-07-001-0.png new file mode 100644 index 0000000..02d17b6 Binary files /dev/null and b/Site/static/images/pokemon/0050-07-001-0.png differ diff --git a/Site/static/images/pokemon/0051-07-001-0.png b/Site/static/images/pokemon/0051-07-001-0.png new file mode 100644 index 0000000..0c75ae0 Binary files /dev/null and b/Site/static/images/pokemon/0051-07-001-0.png differ diff --git a/Site/static/images/pokemon/0052-07-001-0.png b/Site/static/images/pokemon/0052-07-001-0.png new file mode 100644 index 0000000..6d0b7ae Binary files /dev/null and b/Site/static/images/pokemon/0052-07-001-0.png differ diff --git a/Site/static/images/pokemon/0052-08-002-0.png b/Site/static/images/pokemon/0052-08-002-0.png new file mode 100644 index 0000000..6ae786d Binary files /dev/null and b/Site/static/images/pokemon/0052-08-002-0.png differ diff --git a/Site/static/images/pokemon/0053-07-001-0.png b/Site/static/images/pokemon/0053-07-001-0.png new file mode 100644 index 0000000..bc2676c Binary files /dev/null and b/Site/static/images/pokemon/0053-07-001-0.png differ diff --git a/Site/static/images/pokemon/0058-08-001-0.png b/Site/static/images/pokemon/0058-08-001-0.png new file mode 100644 index 0000000..b90342d Binary files /dev/null and b/Site/static/images/pokemon/0058-08-001-0.png differ diff --git a/Site/static/images/pokemon/0059-08-001-0.png b/Site/static/images/pokemon/0059-08-001-0.png new file mode 100644 index 0000000..e44b359 Binary files /dev/null and b/Site/static/images/pokemon/0059-08-001-0.png differ diff --git a/Site/static/images/pokemon/0074-07-001-0.png b/Site/static/images/pokemon/0074-07-001-0.png new file mode 100644 index 0000000..5b5bcc4 Binary files /dev/null and b/Site/static/images/pokemon/0074-07-001-0.png differ diff --git a/Site/static/images/pokemon/0075-07-001-0.png b/Site/static/images/pokemon/0075-07-001-0.png new file mode 100644 index 0000000..b706c27 Binary files /dev/null and b/Site/static/images/pokemon/0075-07-001-0.png differ diff --git a/Site/static/images/pokemon/0076-07-001-0.png b/Site/static/images/pokemon/0076-07-001-0.png new file mode 100644 index 0000000..1ad2409 Binary files /dev/null and b/Site/static/images/pokemon/0076-07-001-0.png differ diff --git a/Site/static/images/pokemon/0077-08-001-0.png b/Site/static/images/pokemon/0077-08-001-0.png new file mode 100644 index 0000000..073b65f Binary files /dev/null and b/Site/static/images/pokemon/0077-08-001-0.png differ diff --git a/Site/static/images/pokemon/0078-08-001-0.png b/Site/static/images/pokemon/0078-08-001-0.png new file mode 100644 index 0000000..05fd125 Binary files /dev/null and b/Site/static/images/pokemon/0078-08-001-0.png differ diff --git a/Site/static/images/pokemon/0079-08-001-0.png b/Site/static/images/pokemon/0079-08-001-0.png new file mode 100644 index 0000000..f21fffc Binary files /dev/null and b/Site/static/images/pokemon/0079-08-001-0.png differ diff --git a/Site/static/images/pokemon/0080-08-001-0.png b/Site/static/images/pokemon/0080-08-001-0.png new file mode 100644 index 0000000..62f38d9 Binary files /dev/null and b/Site/static/images/pokemon/0080-08-001-0.png differ diff --git a/Site/static/images/pokemon/0083-08-001-0.png b/Site/static/images/pokemon/0083-08-001-0.png new file mode 100644 index 0000000..2ddadb9 Binary files /dev/null and b/Site/static/images/pokemon/0083-08-001-0.png differ diff --git a/Site/static/images/pokemon/0088-07-001-0.png b/Site/static/images/pokemon/0088-07-001-0.png new file mode 100644 index 0000000..fb1bffc Binary files /dev/null and b/Site/static/images/pokemon/0088-07-001-0.png differ diff --git a/Site/static/images/pokemon/0089-07-001-0.png b/Site/static/images/pokemon/0089-07-001-0.png new file mode 100644 index 0000000..3c8222f Binary files /dev/null and b/Site/static/images/pokemon/0089-07-001-0.png differ diff --git a/Site/static/images/pokemon/0100-08-001-0.png b/Site/static/images/pokemon/0100-08-001-0.png new file mode 100644 index 0000000..bc18707 Binary files /dev/null and b/Site/static/images/pokemon/0100-08-001-0.png differ diff --git a/Site/static/images/pokemon/0101-08-001-0.png b/Site/static/images/pokemon/0101-08-001-0.png new file mode 100644 index 0000000..99dbec0 Binary files /dev/null and b/Site/static/images/pokemon/0101-08-001-0.png differ diff --git a/Site/static/images/pokemon/0103-07-001-0.png b/Site/static/images/pokemon/0103-07-001-0.png new file mode 100644 index 0000000..02127d3 Binary files /dev/null and b/Site/static/images/pokemon/0103-07-001-0.png differ diff --git a/Site/static/images/pokemon/0105-07-001-0.png b/Site/static/images/pokemon/0105-07-001-0.png new file mode 100644 index 0000000..1ff0898 Binary files /dev/null and b/Site/static/images/pokemon/0105-07-001-0.png differ diff --git a/Site/static/images/pokemon/0110-08-001-0.png b/Site/static/images/pokemon/0110-08-001-0.png new file mode 100644 index 0000000..b5248f0 Binary files /dev/null and b/Site/static/images/pokemon/0110-08-001-0.png differ diff --git a/Site/static/images/pokemon/0122-08-001-0.png b/Site/static/images/pokemon/0122-08-001-0.png new file mode 100644 index 0000000..733b0b0 Binary files /dev/null and b/Site/static/images/pokemon/0122-08-001-0.png differ diff --git a/Site/static/images/pokemon/0128-09-001-0.png b/Site/static/images/pokemon/0128-09-001-0.png new file mode 100644 index 0000000..ff08b6d Binary files /dev/null and b/Site/static/images/pokemon/0128-09-001-0.png differ diff --git a/Site/static/images/pokemon/0128-09-002-0.png b/Site/static/images/pokemon/0128-09-002-0.png new file mode 100644 index 0000000..900335c Binary files /dev/null and b/Site/static/images/pokemon/0128-09-002-0.png differ diff --git a/Site/static/images/pokemon/0128-09-003-0.png b/Site/static/images/pokemon/0128-09-003-0.png new file mode 100644 index 0000000..4ee198d Binary files /dev/null and b/Site/static/images/pokemon/0128-09-003-0.png differ diff --git a/Site/static/images/pokemon/0144-08-001-0.png b/Site/static/images/pokemon/0144-08-001-0.png new file mode 100644 index 0000000..2f0b39b Binary files /dev/null and b/Site/static/images/pokemon/0144-08-001-0.png differ diff --git a/Site/static/images/pokemon/0145-08-001-0.png b/Site/static/images/pokemon/0145-08-001-0.png new file mode 100644 index 0000000..7a2c2fc Binary files /dev/null and b/Site/static/images/pokemon/0145-08-001-0.png differ diff --git a/Site/static/images/pokemon/0146-08-001-0.png b/Site/static/images/pokemon/0146-08-001-0.png new file mode 100644 index 0000000..77c155f Binary files /dev/null and b/Site/static/images/pokemon/0146-08-001-0.png differ diff --git a/Site/static/images/pokemon/0157-08-001-0.png b/Site/static/images/pokemon/0157-08-001-0.png new file mode 100644 index 0000000..5cf3ac5 Binary files /dev/null and b/Site/static/images/pokemon/0157-08-001-0.png differ diff --git a/Site/static/images/pokemon/0194-09-001-0.png b/Site/static/images/pokemon/0194-09-001-0.png new file mode 100644 index 0000000..7beee4e Binary files /dev/null and b/Site/static/images/pokemon/0194-09-001-0.png differ diff --git a/Site/static/images/pokemon/0199-08-001-0.png b/Site/static/images/pokemon/0199-08-001-0.png new file mode 100644 index 0000000..f435d0e Binary files /dev/null and b/Site/static/images/pokemon/0199-08-001-0.png differ diff --git a/Site/static/images/pokemon/0211-08-001-0.png b/Site/static/images/pokemon/0211-08-001-0.png new file mode 100644 index 0000000..3afd7a6 Binary files /dev/null and b/Site/static/images/pokemon/0211-08-001-0.png differ diff --git a/Site/static/images/pokemon/0215-08-000-1.png b/Site/static/images/pokemon/0215-08-000-1.png new file mode 100644 index 0000000..4f0ff52 Binary files /dev/null and b/Site/static/images/pokemon/0215-08-000-1.png differ diff --git a/Site/static/images/pokemon/0215-08-000-2.png b/Site/static/images/pokemon/0215-08-000-2.png new file mode 100644 index 0000000..5475338 Binary files /dev/null and b/Site/static/images/pokemon/0215-08-000-2.png differ diff --git a/Site/static/images/pokemon/0222-08-001-0.png b/Site/static/images/pokemon/0222-08-001-0.png new file mode 100644 index 0000000..69fd3c2 Binary files /dev/null and b/Site/static/images/pokemon/0222-08-001-0.png differ diff --git a/Site/static/images/pokemon/0263-08-001-0.png b/Site/static/images/pokemon/0263-08-001-0.png new file mode 100644 index 0000000..8e17234 Binary files /dev/null and b/Site/static/images/pokemon/0263-08-001-0.png differ diff --git a/Site/static/images/pokemon/0264-08-001-0.png b/Site/static/images/pokemon/0264-08-001-0.png new file mode 100644 index 0000000..8bf34b1 Binary files /dev/null and b/Site/static/images/pokemon/0264-08-001-0.png differ diff --git a/Site/static/images/pokemon/0503-08-001-0.png b/Site/static/images/pokemon/0503-08-001-0.png new file mode 100644 index 0000000..fc3b838 Binary files /dev/null and b/Site/static/images/pokemon/0503-08-001-0.png differ diff --git a/Site/static/images/pokemon/0549-08-001-0.png b/Site/static/images/pokemon/0549-08-001-0.png new file mode 100644 index 0000000..d363c8c Binary files /dev/null and b/Site/static/images/pokemon/0549-08-001-0.png differ diff --git a/Site/static/images/pokemon/0554-08-001-0.png b/Site/static/images/pokemon/0554-08-001-0.png new file mode 100644 index 0000000..c454aaa Binary files /dev/null and b/Site/static/images/pokemon/0554-08-001-0.png differ diff --git a/Site/static/images/pokemon/0562-08-001-0.png b/Site/static/images/pokemon/0562-08-001-0.png new file mode 100644 index 0000000..c445702 Binary files /dev/null and b/Site/static/images/pokemon/0562-08-001-0.png differ diff --git a/Site/static/images/pokemon/0570-08-001-0.png b/Site/static/images/pokemon/0570-08-001-0.png new file mode 100644 index 0000000..6883289 Binary files /dev/null and b/Site/static/images/pokemon/0570-08-001-0.png differ diff --git a/Site/static/images/pokemon/0571-08-001-0.png b/Site/static/images/pokemon/0571-08-001-0.png new file mode 100644 index 0000000..267e413 Binary files /dev/null and b/Site/static/images/pokemon/0571-08-001-0.png differ diff --git a/Site/static/images/pokemon/0618-08-001-0.png b/Site/static/images/pokemon/0618-08-001-0.png new file mode 100644 index 0000000..ca6b9ad Binary files /dev/null and b/Site/static/images/pokemon/0618-08-001-0.png differ diff --git a/Site/static/images/pokemon/0628-08-001-0.png b/Site/static/images/pokemon/0628-08-001-0.png new file mode 100644 index 0000000..f525315 Binary files /dev/null and b/Site/static/images/pokemon/0628-08-001-0.png differ diff --git a/Site/static/images/pokemon/0706-08-001-0.png b/Site/static/images/pokemon/0706-08-001-0.png new file mode 100644 index 0000000..7eb27ef Binary files /dev/null and b/Site/static/images/pokemon/0706-08-001-0.png differ diff --git a/Site/static/images/pokemon/0713-08-001-0.png b/Site/static/images/pokemon/0713-08-001-0.png new file mode 100644 index 0000000..2fe55f2 Binary files /dev/null and b/Site/static/images/pokemon/0713-08-001-0.png differ diff --git a/images-new/0019-07-001-0.png b/images-new/0019-07-001-0.png new file mode 100644 index 0000000..1295b7c Binary files /dev/null and b/images-new/0019-07-001-0.png differ diff --git a/images-new/0020-07-001-0.png b/images-new/0020-07-001-0.png new file mode 100644 index 0000000..1a246a0 Binary files /dev/null and b/images-new/0020-07-001-0.png differ diff --git a/images-new/0026-07-001-0.png b/images-new/0026-07-001-0.png new file mode 100644 index 0000000..2c40ef3 Binary files /dev/null and b/images-new/0026-07-001-0.png differ diff --git a/images-new/0027-07-001-0.png b/images-new/0027-07-001-0.png new file mode 100644 index 0000000..91cf7c4 Binary files /dev/null and b/images-new/0027-07-001-0.png differ diff --git a/images-new/0028-07-001-0.png b/images-new/0028-07-001-0.png new file mode 100644 index 0000000..e9de252 Binary files /dev/null and b/images-new/0028-07-001-0.png differ diff --git a/images-new/0037-07-001-0.png b/images-new/0037-07-001-0.png new file mode 100644 index 0000000..e48335c Binary files /dev/null and b/images-new/0037-07-001-0.png differ diff --git a/images-new/0038-07-001-0.png b/images-new/0038-07-001-0.png new file mode 100644 index 0000000..00e8428 Binary files /dev/null and b/images-new/0038-07-001-0.png differ diff --git a/images-new/0050-07-001-0.png b/images-new/0050-07-001-0.png new file mode 100644 index 0000000..02d17b6 Binary files /dev/null and b/images-new/0050-07-001-0.png differ diff --git a/images-new/0051-07-001-0.png b/images-new/0051-07-001-0.png new file mode 100644 index 0000000..0c75ae0 Binary files /dev/null and b/images-new/0051-07-001-0.png differ diff --git a/images-new/0052-07-001-0.png b/images-new/0052-07-001-0.png new file mode 100644 index 0000000..6d0b7ae Binary files /dev/null and b/images-new/0052-07-001-0.png differ diff --git a/images-new/0052-08-002-0.png b/images-new/0052-08-002-0.png new file mode 100644 index 0000000..6ae786d Binary files /dev/null and b/images-new/0052-08-002-0.png differ diff --git a/images-new/0053-07-001-0.png b/images-new/0053-07-001-0.png new file mode 100644 index 0000000..bc2676c Binary files /dev/null and b/images-new/0053-07-001-0.png differ diff --git a/images-new/0058-08-001-0.png b/images-new/0058-08-001-0.png new file mode 100644 index 0000000..b90342d Binary files /dev/null and b/images-new/0058-08-001-0.png differ diff --git a/images-new/0059-08-001-0.png b/images-new/0059-08-001-0.png new file mode 100644 index 0000000..e44b359 Binary files /dev/null and b/images-new/0059-08-001-0.png differ diff --git a/images-new/0074-07-001-0.png b/images-new/0074-07-001-0.png new file mode 100644 index 0000000..5b5bcc4 Binary files /dev/null and b/images-new/0074-07-001-0.png differ diff --git a/images-new/0075-07-001-0.png b/images-new/0075-07-001-0.png new file mode 100644 index 0000000..b706c27 Binary files /dev/null and b/images-new/0075-07-001-0.png differ diff --git a/images-new/0076-07-001-0.png b/images-new/0076-07-001-0.png new file mode 100644 index 0000000..1ad2409 Binary files /dev/null and b/images-new/0076-07-001-0.png differ diff --git a/images-new/0077-08-001-0.png b/images-new/0077-08-001-0.png new file mode 100644 index 0000000..073b65f Binary files /dev/null and b/images-new/0077-08-001-0.png differ diff --git a/images-new/0078-08-001-0.png b/images-new/0078-08-001-0.png new file mode 100644 index 0000000..05fd125 Binary files /dev/null and b/images-new/0078-08-001-0.png differ diff --git a/images-new/0079-08-001-0.png b/images-new/0079-08-001-0.png new file mode 100644 index 0000000..f21fffc Binary files /dev/null and b/images-new/0079-08-001-0.png differ diff --git a/images-new/0080-08-001-0.png b/images-new/0080-08-001-0.png new file mode 100644 index 0000000..62f38d9 Binary files /dev/null and b/images-new/0080-08-001-0.png differ diff --git a/images-new/0083-08-001-0.png b/images-new/0083-08-001-0.png new file mode 100644 index 0000000..2ddadb9 Binary files /dev/null and b/images-new/0083-08-001-0.png differ diff --git a/images-new/0088-07-001-0.png b/images-new/0088-07-001-0.png new file mode 100644 index 0000000..fb1bffc Binary files /dev/null and b/images-new/0088-07-001-0.png differ diff --git a/images-new/0089-07-001-0.png b/images-new/0089-07-001-0.png new file mode 100644 index 0000000..3c8222f Binary files /dev/null and b/images-new/0089-07-001-0.png differ diff --git a/images-new/0100-08-001-0.png b/images-new/0100-08-001-0.png new file mode 100644 index 0000000..bc18707 Binary files /dev/null and b/images-new/0100-08-001-0.png differ diff --git a/images-new/0101-08-001-0.png b/images-new/0101-08-001-0.png new file mode 100644 index 0000000..99dbec0 Binary files /dev/null and b/images-new/0101-08-001-0.png differ diff --git a/images-new/0103-07-001-0.png b/images-new/0103-07-001-0.png new file mode 100644 index 0000000..02127d3 Binary files /dev/null and b/images-new/0103-07-001-0.png differ diff --git a/images-new/0105-07-001-0.png b/images-new/0105-07-001-0.png new file mode 100644 index 0000000..1ff0898 Binary files /dev/null and b/images-new/0105-07-001-0.png differ diff --git a/images-new/0110-08-001-0.png b/images-new/0110-08-001-0.png new file mode 100644 index 0000000..b5248f0 Binary files /dev/null and b/images-new/0110-08-001-0.png differ diff --git a/images-new/0122-08-001-0.png b/images-new/0122-08-001-0.png new file mode 100644 index 0000000..733b0b0 Binary files /dev/null and b/images-new/0122-08-001-0.png differ diff --git a/images-new/0128-09-001-0.png b/images-new/0128-09-001-0.png new file mode 100644 index 0000000..ff08b6d Binary files /dev/null and b/images-new/0128-09-001-0.png differ diff --git a/images-new/0128-09-002-0.png b/images-new/0128-09-002-0.png new file mode 100644 index 0000000..900335c Binary files /dev/null and b/images-new/0128-09-002-0.png differ diff --git a/images-new/0128-09-003-0.png b/images-new/0128-09-003-0.png new file mode 100644 index 0000000..4ee198d Binary files /dev/null and b/images-new/0128-09-003-0.png differ diff --git a/images-new/0144-08-001-0.png b/images-new/0144-08-001-0.png new file mode 100644 index 0000000..2f0b39b Binary files /dev/null and b/images-new/0144-08-001-0.png differ diff --git a/images-new/0145-08-001-0.png b/images-new/0145-08-001-0.png new file mode 100644 index 0000000..7a2c2fc Binary files /dev/null and b/images-new/0145-08-001-0.png differ diff --git a/images-new/0146-08-001-0.png b/images-new/0146-08-001-0.png new file mode 100644 index 0000000..77c155f Binary files /dev/null and b/images-new/0146-08-001-0.png differ diff --git a/images-new/0157-08-001-0.png b/images-new/0157-08-001-0.png new file mode 100644 index 0000000..5cf3ac5 Binary files /dev/null and b/images-new/0157-08-001-0.png differ diff --git a/images-new/0194-09-001-0.png b/images-new/0194-09-001-0.png new file mode 100644 index 0000000..7beee4e Binary files /dev/null and b/images-new/0194-09-001-0.png differ diff --git a/images-new/0199-08-001-0.png b/images-new/0199-08-001-0.png new file mode 100644 index 0000000..f435d0e Binary files /dev/null and b/images-new/0199-08-001-0.png differ diff --git a/images-new/0211-08-001-0.png b/images-new/0211-08-001-0.png new file mode 100644 index 0000000..3afd7a6 Binary files /dev/null and b/images-new/0211-08-001-0.png differ diff --git a/images-new/0215-08-000-1.png b/images-new/0215-08-000-1.png new file mode 100644 index 0000000..4f0ff52 Binary files /dev/null and b/images-new/0215-08-000-1.png differ diff --git a/images-new/0215-08-000-2.png b/images-new/0215-08-000-2.png new file mode 100644 index 0000000..5475338 Binary files /dev/null and b/images-new/0215-08-000-2.png differ diff --git a/images-new/0222-08-001-0.png b/images-new/0222-08-001-0.png new file mode 100644 index 0000000..69fd3c2 Binary files /dev/null and b/images-new/0222-08-001-0.png differ diff --git a/images-new/0263-08-001-0.png b/images-new/0263-08-001-0.png new file mode 100644 index 0000000..8e17234 Binary files /dev/null and b/images-new/0263-08-001-0.png differ diff --git a/images-new/0264-08-001-0.png b/images-new/0264-08-001-0.png new file mode 100644 index 0000000..8bf34b1 Binary files /dev/null and b/images-new/0264-08-001-0.png differ diff --git a/images-new/0503-08-001-0.png b/images-new/0503-08-001-0.png new file mode 100644 index 0000000..fc3b838 Binary files /dev/null and b/images-new/0503-08-001-0.png differ diff --git a/images-new/0549-08-001-0.png b/images-new/0549-08-001-0.png new file mode 100644 index 0000000..d363c8c Binary files /dev/null and b/images-new/0549-08-001-0.png differ diff --git a/images-new/0554-08-001-0.png b/images-new/0554-08-001-0.png new file mode 100644 index 0000000..c454aaa Binary files /dev/null and b/images-new/0554-08-001-0.png differ diff --git a/images-new/0562-08-001-0.png b/images-new/0562-08-001-0.png new file mode 100644 index 0000000..c445702 Binary files /dev/null and b/images-new/0562-08-001-0.png differ diff --git a/images-new/0570-08-001-0.png b/images-new/0570-08-001-0.png new file mode 100644 index 0000000..6883289 Binary files /dev/null and b/images-new/0570-08-001-0.png differ diff --git a/images-new/0571-08-001-0.png b/images-new/0571-08-001-0.png new file mode 100644 index 0000000..267e413 Binary files /dev/null and b/images-new/0571-08-001-0.png differ diff --git a/images-new/0618-08-001-0.png b/images-new/0618-08-001-0.png new file mode 100644 index 0000000..ca6b9ad Binary files /dev/null and b/images-new/0618-08-001-0.png differ diff --git a/images-new/0628-08-001-0.png b/images-new/0628-08-001-0.png new file mode 100644 index 0000000..f525315 Binary files /dev/null and b/images-new/0628-08-001-0.png differ diff --git a/images-new/0706-08-001-0.png b/images-new/0706-08-001-0.png new file mode 100644 index 0000000..7eb27ef Binary files /dev/null and b/images-new/0706-08-001-0.png differ diff --git a/images-new/0713-08-001-0.png b/images-new/0713-08-001-0.png new file mode 100644 index 0000000..2fe55f2 Binary files /dev/null and b/images-new/0713-08-001-0.png differ