Merge with master

DBEditor/DBEditor.py

@@ -20,8 +20,13 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from db_controller import DBController
 from pokemon_db_ui import PokemonUI
 from DataGatherers.update_location_information import process_pokemon_for_location_data
+from DataGatherers.pokemondb_scraper import retrieve_all_pokemon_forms
+from DataGatherers.update_storable_in_home import update_storable_in_home
+from DataGatherers.update_location_information import profile_update_location_information
+from DataGatherers.Update_evolution_information import update_evolution_chains
 from DataGatherers.cache_manager import CacheManager
 from event_system import event_system
+from db_classes import PokemonForm
 
 import logging
 
@@ -40,7 +45,6 @@ class PokemonFormGatherer(QThread):
     def run(self):
         cache = CacheManager()
         debugpy.debug_this_thread()
-        from DataGatherers.pokemondb_scraper import retrieve_all_pokemon_forms
         retrieve_all_pokemon_forms(cache, progress_callback=self.progress_signal.emit)
         self.finished_signal.emit()
 
@@ -51,7 +55,6 @@ class PokemonHomeStatusUpdater(QThread):
     def run(self):
         cache = CacheManager()
         debugpy.debug_this_thread()
-        from DataGatherers.update_storable_in_home import update_storable_in_home
         update_storable_in_home(cache, progress_callback=self.progress_signal.emit)
         self.finished_signal.emit()
 
@@ -61,10 +64,8 @@ class PokemonEvolutionUpdater(QThread):
 
     def run(self):
         time.sleep(0.1)
-        #pdb.set_trace()
         debugpy.debug_this_thread()
         cache = CacheManager()
-        from DataGatherers.Update_evolution_information import update_evolution_chains
         update_evolution_chains(cache, progress_callback=self.progress_signal.emit)
         self.finished_signal.emit()
 
@@ -74,10 +75,9 @@ class PokemonEncounterUpdater(QThread):
 
     def run(self):
         time.sleep(0.1)
-        debugpy.debug_this_thread()
+        #debugpy.debug_this_thread()
         cache = CacheManager()
-        from DataGatherers.update_location_information import update_location_information
-        update_location_information(cache, progress_callback=self.progress_signal.emit)
+        profile_update_location_information(cache, progress_callback=self.progress_signal.emit)
         self.finished_signal.emit()
 
 class DBEditor(QMainWindow):
@@ -437,6 +437,28 @@ class DBEditor(QMainWindow):
             self.logger.info("Database reinitialized successfully.")
             QMessageBox.information(self, 'Database Reinitialized', 'The database has been cleared and reinitialized.')
 
+    def determine_origin_mark(self, pfic, target_generation):
+        encounters = event_system.call_sync('get_encounter_locations', pfic)
+        if encounters:
+            generation_encounters = []
+            for encounter in encounters:
+                game_generation = event_system.call_sync('get_game_generation', encounter[0])
+                game_id = event_system.call_sync('get_game_id_for_name', encounter[0])  
+                encounter = encounter + (game_generation, game_id)
+                if game_generation == target_generation:
+                    generation_encounters.append(encounter)
+            if len(generation_encounters) > 0:
+                generation_encounters = sorted(generation_encounters, key=lambda x: x[12])
+                form_info = event_system.call_sync('get_pokemon_data', pfic)
+                mark_id = event_system.call_sync('get_mark_for_game_name', generation_encounters[0][0])
+                if mark_id == None:
+                    self.logger.info(f"No mark found for {form_info[0]} {form_info[1]}")
+                else:
+                    mark_details = event_system.call_sync('get_mark_details', mark_id)
+                    self.logger.info(f"Mark for {form_info[0]} {form_info[1]} is {mark_details[0]} - {mark_details[1]}")
+                    return mark_id
+        return None
+
     def gather_marks_info(self, data):
         event_system.emit_sync('clear_log_display')
         self.logger.info("Starting to gather marks information...")
@@ -444,31 +466,119 @@ class DBEditor(QMainWindow):
         pokemon_list = event_system.call_sync('get_pokemon_list')
 
         for pfic, name, form_name, national_dex in pokemon_list:
-            pokemon_data = event_system.call_sync('get_pokemon_data', pfic)
+            pokemon_data = event_system.call_sync('get_pokemon_form_by_pfic', pfic)
+
+            if pokemon_data.storable_in_home == False:
+                continue;
+            
+            self.logger.info(f"Determining mark for {pokemon_data.name} {pokemon_data.form_name if pokemon_data.form_name else ''}")
+
+            target_generation = pokemon_data.generation
+
             #Rule 1
             # 1. If a pokemon form has a previous evolution from within the same generation, 
             # use the mark of the previous evolution. This should be recursive within the same generation.
             chain = event_system.call_sync('get_evolution_chain', pfic)
             if chain:
-                find_me = lambda x: x[0] == pfic
-                target_index = next((i for i, item in enumerate(chain) if find_me(item)), -1)
                 base_form_in_generation = None
+                last_pfic = pfic
                 current_pfic = pfic
                 while True:
-                    current_pfic = event_system.call_sync('get_evolution_parent', current_pfic)
+                    current_pfic = event_system.call_sync('get_evolution_parent', last_pfic)
                     if current_pfic == None:
-                        base_form_in_generation = current_pfic
+                        base_form_in_generation = last_pfic
                         break
-                    chain_pokemon_data = event_system.call_sync('get_pokemon_data', current_pfic)
-                    if chain_pokemon_data[3] == pokemon_data[3]:
+                    chain_pokemon_data = event_system.call_sync('get_pokemon_form_by_pfic', current_pfic[0])
+                    if chain_pokemon_data.generation == target_generation:
                         base_form_in_generation = current_pfic
+                    else:
+                        base_form_in_generation = last_pfic
                         break
+                    last_pfic = current_pfic[0]
 
-                if base_form_in_generation:
+                if base_form_in_generation and base_form_in_generation != pfic:
                     self.logger.info(f"Base form in generation for {name} {form_name if form_name else ''} is {base_form_in_generation}")
-
-
-
+                    mark_id = self.determine_origin_mark(base_form_in_generation, target_generation)
+                    if mark_id != None:
+                        event_system.emit_sync('assign_mark_to_form', (pfic, mark_id))
+                        continue;
+                elif base_form_in_generation == pfic:
+                    mark_id = self.determine_origin_mark(pfic, target_generation)
+                    if mark_id != None:
+                        event_system.emit_sync('assign_mark_to_form', (pfic, mark_id))
+                        continue;
+            #Rule 2a
+            # If a pokemon form has no previous evolution from within the same generation, 
+            # look at the encounters of the pokemon form from this generation and use the mark of the earliest
+            # game you can encounter that form in from that generation
+            mark_id = self.determine_origin_mark(pfic, target_generation)
+            if mark_id != None:
+                event_system.emit_sync('assign_mark_to_form', (pfic, mark_id))
+                continue;
+
+            #Rule 3
+            # If there are no encounters for the pokemon form from this generation,
+            # look to see if a previous evolution has an encounter from this generation, and use the mark of the earliest
+            # game from this generation that the previous evolution is encounterable in.
+            evolve_encounters = event_system.call_sync('get_evolve_encounters', pfic)
+            if evolve_encounters:
+                available_encounters = []
+                for encounter in evolve_encounters:
+                    game_id = encounter.game_id
+                    game_info = event_system.call_sync('get_game_by_id', game_id)
+                    if game_info[2] == target_generation:
+                        available_encounters.append(encounter)
+
+                if len(available_encounters) > 0:
+                    available_encounters = sorted(available_encounters, key=lambda x: x.game_id)
+                    mark_id = self.determine_origin_mark(available_encounters[0].from_pfic, target_generation)
+                    if mark_id != None:
+                        event_system.emit_sync('assign_mark_to_form', (pfic, mark_id))
+                        continue;
+            
+            encounters = event_system.call_sync('get_encounter_locations', pfic)
+            count = 0
+            if encounters:
+                for encounter in encounters:
+                    game_id = event_system.call_sync('get_game_id_for_name', encounter[0])
+                    game_generation = event_system.call_sync('get_game_generation', game_id)
+                    if game_generation == target_generation:
+                        count += 1
+            if count == 0:
+                #Rule 2b
+                # Check to see if this is a sub-form pokemon, and if so, use the mark of the base form.
+                base = pfic[:-6]
+                alternate_forms = event_system.call_sync('find_default_form', base)
+                if alternate_forms:
+                    form_found = False
+                    for alternate_form in alternate_forms:
+                        mark_id = self.determine_origin_mark(alternate_form.pfic, target_generation)
+                        if mark_id != None:
+                            event_system.emit_sync('assign_mark_to_form', (pfic, mark_id))
+                            form_found = True
+                            break;
+                    if form_found:
+                        continue;
+
+            #Rule 4
+            # If there are no encounters for the pokemon form or its evolution line from this generation,
+            # use the mark of the earliest game of the generation is marked as being introducted in.
+            if encounters:
+                earliest_game = 100
+                for encounter in encounters:
+                    game_id = event_system.call_sync('get_game_id_for_name', encounter[0])
+                    if game_id <= earliest_game:
+                        earliest_game = game_id
+                if earliest_game < 100:
+                    form_info = event_system.call_sync('get_pokemon_data', pfic)
+                    mark_id = event_system.call_sync('get_mark_for_game', earliest_game)
+                    if mark_id == None:
+                        self.logger.info(f"No mark found for {form_info[0]} {form_info[1]}")
+                    else:
+                        mark_details = event_system.call_sync('get_mark_details', mark_id)
+                        self.logger.info(f"Mark for {form_info[0]} {form_info[1]} is {mark_details[0]} - {mark_details[1]}")
+                        event_system.emit_sync('assign_mark_to_form', (pfic, mark_id))
+                    continue;
 
             
 

DBEditor/db_classes.py

@@ -0,0 +1,18 @@
+
+class PokemonForm:
+    def __init__(self, pfic: int, name: str, form_name: str, national_dex: int, generation: int, is_baby_form: bool, storable_in_home: bool):
+        self.pfic = pfic
+        self.name = name
+        self.form_name = form_name
+        self.national_dex = national_dex
+        self.generation = generation
+        self.is_baby_form = is_baby_form
+        self.storable_in_home = storable_in_home
+
+class EvolveEncounter:
+    def __init__(self, pfic: int, game_id: int, day: str, time: str, from_pfic: int):
+        self.pfic = pfic
+        self.game_id = game_id
+        self.day = day
+        self.time = time
+        self.from_pfic = from_pfic

DBEditor/db_controller.py

@@ -1,16 +1,30 @@
 import sqlite3
 import sys
 import os
+import threading
+from queue import Queue
+from concurrent.futures import ThreadPoolExecutor
 
 from event_system import event_system
+from db_classes import PokemonForm, EvolveEncounter
 
 def parse_pfic(pfic):
     parts = pfic.split('-')
     return tuple(int(part) if part.isdigit() else part for part in parts)
 
 class DBController:
-    def __init__(self):
-        self.conn = sqlite3.connect(':memory:', check_same_thread=False)  # Use in-memory database for runtime
+    def __init__(self, db_path=':memory:', max_connections=10):
+        self.db_path = db_path
+        self.connection_pool = Queue(maxsize=max_connections)
+        self.lock = threading.Lock()
+
+        for _ in range(max_connections):
+            conn = sqlite3.connect(db_path, check_same_thread=False)
+            self.connection_pool.put(conn)
+
+        self.executor = ThreadPoolExecutor(max_workers=max_connections)
+
+        self.conn = sqlite3.connect(db_path, check_same_thread=False)
         self.cursor = self.conn.cursor()
 
         event_system.add_listener('get_pokemon_data', self.load_pokemon_details)
@@ -35,6 +49,13 @@ class DBController:
         event_system.add_listener('save_changes', self.save_changes)
         event_system.add_listener('export_database', self.export_database)
         event_system.add_listener('assign_mark_to_form', self.assign_mark_to_form)
+        event_system.add_listener('get_mark_for_game', self.get_mark_for_game)
+        event_system.add_listener('get_mark_for_game_name', self.get_mark_for_game_name)
+        event_system.add_listener('get_mark_details', self.get_mark_details)
+        event_system.add_listener('get_evolve_encounters', self.get_evolve_encounters)
+        event_system.add_listener('get_game_by_id', self.get_game_by_id)
+        event_system.add_listener('get_pokemon_form_by_pfic', self.get_pokemon_form_by_pfic)
+        event_system.add_listener('find_default_form', self.find_default_form)
 
     def init_database(self):
         disk_conn = sqlite3.connect('pokemon_forms.db')
@@ -50,6 +71,7 @@ class DBController:
         self.create_encounters_table(disk_cursor)
         self.create_mark_table(disk_cursor)
         self.create_form_marks_table(disk_cursor)
+        self.create_evolve_encounter_table(disk_cursor)
 
         # Commit changes to the file-based database
         disk_conn.commit()
@@ -57,6 +79,16 @@ class DBController:
         # Copy the file-based database to the in-memory database
         disk_conn.backup(self.conn)
 
+        temp_connections = []
+        for _ in range(self.connection_pool.qsize()):
+            conn = self.connection_pool.get()
+            disk_conn.backup(conn)
+            temp_connections.append(conn)
+
+        # Put all connections back into the pool
+        for conn in temp_connections:
+            self.connection_pool.put(conn)
+
         # Close the file-based database connection
         disk_conn.close()
 
@@ -165,6 +197,7 @@ class DBController:
             ("Sinnoh", "images/marks/BDSP_icon_HOME.png", ["Brilliant Diamond", "Shining Pearl"]),
             ("Hisui", "images/marks/Arceus_mark_HOME.png", ["Legends Arceus"]),
             ("Paldea", "images/marks/Paldea_icon_HOME.png", ["Scarlet", "Violet", "The Teal Mask", "The Hidden Treasure of Area Zero"]),
+            ("Markless", "images/marks/Markless_icon_HOME.png", ["Ruby", "Sapphire", "Emerald", "FireRed", "LeafGreen", "Diamond", "Pearl", "Platinum", "HeartGold", "SoulSilver", "Black", "White", "Black 2", "White 2"]),
         ]
         
         # Check if marks already exist
@@ -298,6 +331,21 @@ class DBController:
         )
         ''')
         
+    def create_evolve_encounter_table(self, cursor):
+        cursor.execute('''
+        CREATE TABLE IF NOT EXISTS evolve_encounters (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            pfic TEXT,
+            game_id INTEGER,
+            day TEXT,
+            time TEXT,
+            from_pfic TEXT,
+            FOREIGN KEY (pfic) REFERENCES pokemon_forms (PFIC),
+            FOREIGN KEY (game_id) REFERENCES games (id),
+            FOREIGN KEY (from_pfic) REFERENCES pokemon_forms (PFIC)
+        )
+        ''')
+
     def load_pokemon_details(self, pfic):
         self.cursor.execute('''
             SELECT pf.name, pf.form_name, pf.national_dex, pf.generation, ps.storable_in_home, pf.is_baby_form
@@ -388,27 +436,21 @@ class DBController:
         result = self.cursor.fetchone()
         return result[0] if result else None
     
-    def get_game_id_for_name(self, game):
-        # First, try to find the game in the main games table
-        self.cursor.execute('''
-            SELECT id 
-            FROM games
-            WHERE name = ?
-        ''', (game,))
-        result = self.cursor.fetchone()
-        
+    def get_game_id_for_name(self, game_name):
+        query = "SELECT id FROM games WHERE name = ?"
+        result = self.execute_query(query, (game_name,))
         if result:
-            return result[0]
-        
-        # If not found, check the alternate_game_names table
-        self.cursor.execute('''
-            SELECT game_id 
-            FROM alternate_game_names
-            WHERE alternate_name = ?
-        ''', (game,))
-        result = self.cursor.fetchone()
-        
-        return result[0] if result else None
+            return result[0][0]
+        
+        # If not found in main table, check alternate_game_names
+        query = """
+        SELECT g.id 
+        FROM games g 
+        JOIN alternate_game_names agn ON g.id = agn.game_id 
+        WHERE agn.alternate_name = ?
+        """
+        result = self.execute_query(query, (game_name,))
+        return result[0][0] if result else None
     
     def get_pokemon_with_encounters(self, data):
         self.cursor.execute('''
@@ -484,6 +526,105 @@ class DBController:
 
     def assign_mark_to_form(self, data):
         pfic, mark_id = data
-        self.cursor.execute('INSERT INTO form_marks (pfic, mark_id) VALUES (?, ?)', (pfic, mark_id))
+        self.cursor.execute('''
+            INSERT OR REPLACE INTO form_marks (pfic, mark_id) 
+            VALUES (?, ?)
+        ''', (pfic, mark_id))
         self.conn.commit()
 
+    def get_mark_for_game(self, data):
+        game_id = data
+        self.cursor.execute('SELECT mark_id FROM mark_game_associations WHERE game_id = ?', (game_id,))
+        result = self.cursor.fetchone()
+        return result[0] if result else None
+    
+    def get_mark_for_game_name(self, data):
+        game_name = data
+        self.cursor.execute('''
+            SELECT m.id 
+            FROM marks m
+            JOIN mark_game_associations mga ON m.id = mga.mark_id
+            JOIN games g ON mga.game_id = g.id
+            WHERE g.name = ?
+        ''', (game_name,))
+        result = self.cursor.fetchone()
+        return result[0] if result else None
+    
+    def get_mark_details(self, data):
+        mark_id = data
+        self.cursor.execute('SELECT name, icon_path FROM marks WHERE id = ?', (mark_id,))
+        return self.cursor.fetchone()
+
+    def get_evolve_encounters(self, data):
+        pfic = data
+        self.cursor.execute('SELECT * FROM evolve_encounters WHERE pfic = ?', (pfic,))
+        evolve_encounters = self.cursor.fetchall()
+        return [EvolveEncounter(encounter[1], encounter[2], encounter[3], encounter[4], encounter[5]) for encounter in evolve_encounters]
+    
+    def get_game_by_id(self, data):
+        game_id = data
+        self.cursor.execute('SELECT * FROM games WHERE id = ?', (game_id,))
+        result = self.cursor.fetchone()
+        return result
+    
+    def get_pokemon_form_by_pfic(self, data) -> PokemonForm:
+        pfic = data
+        self.cursor.execute('''
+            SELECT pf.name, pf.form_name, pf.national_dex, pf.generation, ps.storable_in_home, pf.is_baby_form
+            FROM pokemon_forms pf
+            LEFT JOIN pokemon_storage ps ON pf.PFIC = ps.PFIC
+            WHERE pf.PFIC = ?
+        ''', (pfic,))
+        result = self.cursor.fetchone()
+        return PokemonForm(pfic, result[0], result[1], result[2], result[3], result[5], result[4])
+    
+    def find_default_form(self, data):
+        search_pfic = data
+        self.cursor.execute('''
+            SELECT pf.PFIC, pf.name, pf.form_name, pf.national_dex, pf.generation, ps.storable_in_home, pf.is_baby_form
+            FROM pokemon_forms pf
+            LEFT JOIN pokemon_storage ps ON pf.PFIC = ps.PFIC
+            WHERE pf.PFIC LIKE ?
+        ''', (search_pfic + "%",))
+        results = self.cursor.fetchall()
+        return [PokemonForm(result[0], result[1], result[2], result[3], result[4], result[6], result[5]) for result in results]
+
+    def get_connection(self):
+        return self.connection_pool.get()
+
+    def release_connection(self, conn):
+        self.connection_pool.put(conn)
+
+    def execute_query_with_commit(self, query, params=None):
+        conn = self.get_connection()
+        try:
+            with conn:
+                cursor = conn.cursor()
+                if params:
+                    cursor.execute(query, params)
+                else:
+                    cursor.execute(query)
+                conn.commit()
+            return cursor.fetchall()
+        finally:
+            self.release_connection(conn)
+
+    def execute_query(self, query, params=None):
+        conn = self.get_connection()
+        try:
+            with conn:
+                cursor = conn.cursor()
+                if params:
+                    cursor.execute(query, params)
+                else:
+                    cursor.execute(query)
+                return cursor.fetchall()
+        finally:
+            self.release_connection(conn)
+
+    def close(self):
+        while not self.connection_pool.empty():
+            conn = self.connection_pool.get()
+            conn.close()
+        self.executor.shutdown()
+

DataGatherers/DetermineOriginGame.py

@@ -19,6 +19,9 @@ import sys
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from DataGatherers.cache_manager import CacheManager
+import concurrent.futures
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from functools import lru_cache
 
 # List of all main series Pokémon games in chronological order, with special games first in each generation
 all_games = [
@@ -28,7 +31,7 @@ all_games = [
     "Platinum", "HeartGold", "SoulSilver", "Diamond", "Pearl",
     "Black 2", "White 2", "Black", "White",
     "X", "Y", "Omega Ruby", "Alpha Sapphire",
-    "Ultra Sun", "Ultra Moon", "Sun", "Moon",
+    "Ultra Sun", "Ultra Moon", "Sun", "Moon", "Let's Go Pikachu", "Let's Go Eevee",
     "Sword", "Shield", "Expansion Pass",
     "Brilliant Diamond", "Shining Pearl",
     "Legends: Arceus",
@@ -371,24 +374,19 @@ def get_pokemon_data_bulbapedia(pokemon_name, cache: CacheManager):
 def split_td_contents(td):
     groups = []
     current_group = []
-
     for content in td.contents:
-        if isinstance(content, Tag) and (content.name == 'br' or content.name == 'p'):
+        if isinstance(content, NavigableString):
+            text = content.strip()
+            if text:
+                current_group.append(content)
+        elif content.name == 'br':
             if current_group:
-                groups.append(BeautifulSoup('', 'html.parser').new_tag('div'))
-                for item in current_group:
-                    groups[-1].append(copy.copy(item))
-                if content.name == 'p':
-                    groups[-1].append(copy.copy(content))
+                groups.append(''.join(str(item) for item in current_group))
                 current_group = []
         else:
             current_group.append(content)
-
     if current_group:
-        groups.append(BeautifulSoup('', 'html.parser').new_tag('div'))
-        for item in current_group:
-            groups[-1].append(copy.copy(item))
-
+        groups.append(''.join(str(item) for item in current_group))
     return groups
 
 def parse_form_information(html_content):
@@ -588,145 +586,137 @@ def compare_forms(a, b):
 
     return False
 
-def get_locations_from_bulbapedia(pokemon_name, form, cache: CacheManager, default_forms = None):
+@lru_cache(maxsize=100)
+def get_parsed_pokemon_page(pokemon_name, cache):
     page_data = get_pokemon_data_bulbapedia(pokemon_name, cache)
-    if not page_data:
+    return BeautifulSoup(page_data, 'html.parser') if page_data else None
+
+def get_locations_from_bulbapedia(pokemon_name, form, cache: CacheManager, default_forms=None):
+    soup = get_parsed_pokemon_page(pokemon_name, cache)
+    if not soup:
         return None
 
-    soup = BeautifulSoup(page_data, 'html.parser')
+    # Try different methods to find the locations table
+    locations_table = None
+    possible_headers = ['Game locations', 'In side games', 'In spin-off games']
     
-    locations_section = soup.find('span', id='Game_locations')
-    if not locations_section:
-        return None
+    for header in possible_headers:
+        span = soup.find('span', id=header.replace(' ', '_'))
+        if span:
+            locations_table = span.find_next('table', class_='roundy')
+            if locations_table:
+                break
 
-    locations_table = locations_section.find_next('table', class_='roundy')
     if not locations_table:
+        print(f"Warning: Couldn't find locations table for {pokemon_name}")
         return None
 
     raw_game_locations = {}
 
-    # Ok so the table is a bit of a mess. It has some nested tables and stuff.
-    # In each row is a nested table with all the games in a generation.
-    # Next is another nexted table, but i can't tell what for.
-    # within that nested table, is another nested table with the games, either the release pair or a single game spanning two columns.
-    # Next to that is another nested table with the locations.
+    # Process game locations
+    for row in locations_table.select('tr'):
+        games = row.select('th')
+        locations = row.select('td')
         
-    generation_tbody = locations_table.find('tbody', recursive=False)
-    generation_rows = generation_tbody.find_all('tr', recursive=False)
-    for generation_row in generation_rows:
-        random_nested_td = generation_row.find('td', recursive=False)   
-        if not random_nested_td:
-            continue
-        random_nested_table = random_nested_td.find('table', recursive=False)
-        if not random_nested_table:
+        if len(games) != len(locations):
             continue
-        random_nested_tbody = random_nested_table.find('tbody', recursive=False)
-        random_nested_rows = random_nested_tbody.find_all('tr', recursive=False)
-
-        for nested_row in random_nested_rows:
-            if 'Generation' in nested_row.get_text(strip=True):
-                continue
 
-            games_container_td = nested_row.find('td', recursive=False)   
-            if not games_container_td:
-                continue
-            games_container_table = games_container_td.find('table', recursive=False)
-            if not games_container_table:
-                continue
-            games_container_tbody = games_container_table.find('tbody', recursive=False)
-            games_container_rows = games_container_tbody.find_all('tr', recursive=False)
-            for games_container_row in games_container_rows:
-                games = games_container_row.find_all('th')
-                for game in games:
-                    raw_game = game.get_text(strip=True)
-                    if raw_game not in all_games:
-                        continue
-                    locations_container_td = games_container_row.find('td', recursive=False)   
-                    if not locations_container_td:
-                        continue
-                    locations_container_table = locations_container_td.find('table', recursive=False)
-                    if not locations_container_table:
-                        continue
-                    locations_container_tbody = locations_container_table.find('tbody', recursive=False)
-                    locations = locations_container_tbody.find_all('td')
-                    for location in locations:
-                        groups = split_td_contents(location)
-                        for group in groups:
-                            if raw_game not in raw_game_locations:
-                                raw_game_locations[raw_game] = []
-                            raw_game_locations[raw_game].append(group)
+        for game, location in zip(games, locations):
+            raw_game = game.get_text(strip=True)
+            if raw_game in all_games:
+                groups = split_td_contents(location)
+                raw_game_locations.setdefault(raw_game, []).extend(groups)
 
+    # Process events
     events_section = soup.find('span', id='In_events')
-    event_tables = {}
-    if events_section:
-        event_header = events_section.parent
+    event_tables = process_event_tables(events_section) if events_section else {}
 
-        variant = ""
-        for sibling in event_header.find_next_siblings():
-            if sibling.name == 'h4' or "held" in sibling.getText(strip=True).lower():
-                break
-            if sibling.name == 'h5':
-                variant = sibling.get_text(strip=True)
-            if sibling.name == 'table':
-                event_tables[variant] = sibling
+    # Process game locations in parallel
+    with ThreadPoolExecutor() as executor:
+        futures = {executor.submit(process_game_locations, raw_game, raw_locations, form, default_forms): raw_game 
+                   for raw_game, raw_locations in raw_game_locations.items()}
         
-    game_locations = {}
-    for raw_game, raw_locations in raw_game_locations.items():
-        if form is None:
-            for raw_location in raw_locations:
-                raw_text = raw_location.get_text()
-                forms = parse_form_information(str(raw_location))
-                if len(forms) > 0:
-                    for form_info in forms:
-                        main_form = form_info["main_form"]
+        game_locations = {}
+        for future in as_completed(futures):
+            raw_game = futures[future]
+            result = future.result()
+            if result:
+                game_locations[raw_game] = result
 
-                        if default_forms and main_form and main_form in default_forms:
-                            main_form = None
+    # Process event tables
+    for variant in event_tables:
+        if (variant == pokemon_name and form is None) or (form and form in variant):
+            process_event_table(event_tables[variant], game_locations)
 
-                        if main_form and (main_form != "All Forms" and main_form != "Kantonian Form" and main_form != "All Sizes"):
-                            continue
+    return game_locations
 
-                        record_location_info(raw_game, game_locations, raw_location, raw_text)
-                else:
-                    record_location_info(raw_game, game_locations, raw_location, raw_text)
-        else:
-            for raw_location in raw_locations:
-                forms = parse_form_information(str(raw_location))
+def process_event_tables(events_section):
+    event_tables = {}
+    if events_section:
+        next_element = events_section.find_next_sibling()
+        while next_element and next_element.name != 'h3':
+            if next_element.name == 'h4':
+                variant = next_element.text.strip()
+                table = next_element.find_next_sibling('table', class_='roundy')
+                if table:
+                    event_tables[variant] = table
+            next_element = next_element.find_next_sibling()
+    return event_tables
+
+def process_event_table(table, game_locations):
+    for row in table.find_all('tr')[1:]:  # Skip header row
+        cells = row.find_all('td')
+        if len(cells) >= 3:
+            game = cells[0].text.strip()
+            location = cells[2].text.strip()
+            if game in all_games:
+                if game not in game_locations:
+                    game_locations[game] = []
+                game_locations[game].append({"location": f"Event: {location}", "tag": str(cells[2])})
+
+def process_game_locations(raw_game, raw_locations, form, default_forms):
+    locations = []
+    
+    for raw_location in raw_locations:
+        raw_text = raw_location
+        forms = parse_form_information(raw_location)
+        if form is None:
+            if len(forms) > 0:
                 for form_info in forms:
                     main_form = form_info["main_form"]
-                    sub_form = form_info["sub_form"]
+                    if default_forms and main_form and main_form in default_forms:
+                        main_form = None
 
-                    if not main_form:
+                    if main_form and (main_form != "All Forms" and main_form != "Kantonian Form" and main_form != "All Sizes"):
                         continue
 
-                    if main_form == "All Forms" or main_form == "Kantonian Form" or main_form == "All Sizes":
-                        main_form = form
+                    locations.append({"location": raw_text, "tag": raw_location})
+            else:
+                locations.append({"location": raw_text, "tag": raw_location})
+        elif len(forms) > 0:
+            for form_info in forms:
+                if form_matches(form_info, form, default_forms):
+                    locations.append({"location": raw_text, "tag": raw_location})
 
-                    main_form_match = compare_forms(form, main_form)
-                    if not main_form_match:
-                        main_form_match = fuzz.partial_ratio(form.lower(), main_form.lower()) >= 80
+    return locations if locations else None
 
-                    sub_form_match = compare_forms(form, sub_form)
-                    if not sub_form_match:
-                        sub_form_match = False if not sub_form else fuzz.partial_ratio(form.lower(), sub_form.lower()) >= 80
+def form_matches(form_info, form, default_forms):
+    main_form = form_info["main_form"]
+    sub_form = form_info["sub_form"]
 
-                    if main_form_match or sub_form_match:
-                        raw_text = raw_location.get_text()
-                        record_location_info(raw_game, game_locations, raw_location, raw_text)
+    if default_forms and main_form and main_form in default_forms:
+        main_form = None
 
-    # For Later
-    for variant in event_tables:
-        if (variant == pokemon_name and form is None)or (form and form in variant):
-            games_container_rows = event_tables[variant].find_all('tr')
-            for game_row in games_container_rows:
-                entries = game_row.find_all('td')
-                if len(entries) > 1:
-                    games_string = entries[0].find('a').get('title')
-                    for game in all_games:
-                        if game in games_string:
-                            record_location_info(game, game_locations, "Event", "Event")
+    if main_form is None:
+        return False
 
-    return game_locations
+    if main_form in ["All Forms", "Kantonian Form", "All Sizes"]:
+        return True
+
+    main_form_match = compare_forms(form, main_form) or fuzz.partial_ratio(form.lower(), main_form.lower()) >= 80
+    sub_form_match = compare_forms(form, sub_form) or (sub_form and fuzz.partial_ratio(form.lower(), sub_form.lower()) >= 80)
+
+    return main_form_match or sub_form_match
 
 def record_location_info(raw_game, game_locations, raw_location, raw_text):
     if raw_game not in game_locations:
@@ -916,3 +906,4 @@ if __name__ == "__main__":
     
     cache.close()
     print(f"Earliest obtainable games and encounter locations determined for {len(pokemon_list)} Pokémon and saved to pokemon_earliest_games.csv")
+

DataGatherers/Update_evolution_information.py

@@ -13,6 +13,8 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from DataGatherers.cache_manager import CacheManager
 from DataGatherers.DetermineOriginGame import get_evolution_data_from_bulbapedia
 
+from db_controller import DBController
+
 logger = logging.getLogger('ui_feedback')
 
 @dataclass
@@ -37,14 +39,29 @@ def create_evolution_table():
     conn.commit()
     return conn
 
-def insert_evolution_info(conn, evolution_info: EvolutionInfo):
-    cursor = conn.cursor()
-    cursor.execute('''
+def sanitize_for_logging(text: str) -> str:
+    # Replace problematic Unicode characters with ASCII equivalents
+    replacements = {
+        '→': '->',
+        '←': '<-',
+        '⇒': '=>',
+        '⇐': '<=',
+        '↗': '-^>',
+        '↖': '<^-',
+        '♀': 'f',
+        '♂': 'm'
+    }
+    for unicode_char, ascii_char in replacements.items():
+        text = text.replace(unicode_char, ascii_char)
+    return text
+
+def insert_evolution_info(db_controller, evolution_info: EvolutionInfo):
+    db_controller.execute_query_with_commit('''
     INSERT OR REPLACE INTO evolution_chains 
     (from_pfic, to_pfic, method)
     VALUES (?, ?, ?)
     ''', (evolution_info.from_pfic, evolution_info.to_pfic, evolution_info.method))
-    conn.commit()
+    logger.info(sanitize_for_logging(f"Adding a link from {evolution_info.from_pfic} to {evolution_info.to_pfic}, via {evolution_info.method}"))
 
 def strip_pokemon_name(pokemon_name: str, form_name: str) -> str:
     """Remove the Pokémon's name from the form name if present."""
@@ -61,10 +78,9 @@ def fuzzy_match_form(form1: str, form2: str, threshold: int = 80) -> bool:
         return form1 == form2
     return fuzz.ratio(form1.lower(), form2.lower()) >= threshold
 
-def get_pokemon_form_by_name(conn, name: str, form: Optional[str] = None, threshold: int = 80, gender: Optional[str] = None) -> Optional[str]:
-    cursor = conn.cursor()
-    cursor.execute('SELECT PFIC, name, form_name FROM pokemon_forms WHERE name = ?', (name,))
-    results = cursor.fetchall()
+def get_pokemon_form_by_name(db_controller, name: str, form: Optional[str] = None, threshold: int = 80, gender: Optional[str] = None) -> Optional[str]:
+    
+    results = db_controller.execute_query('SELECT PFIC, name, form_name FROM pokemon_forms WHERE name = ?', (name,))
 
     if not results:
         return None
@@ -74,12 +90,12 @@ def get_pokemon_form_by_name(conn, name: str, form: Optional[str] = None, thresh
             if results[0][2] == None:
                 return results[0][0]
             else:
-                return get_pokemon_form_by_name(conn, name, "Male", threshold=100, gender=gender)
+                return get_pokemon_form_by_name(db_controller, name, "Male", threshold=100, gender=gender)
         else:
             return results[0][0]  # Return the PFIC of the first result if no form is specified
     
     if gender:
-        gendered_form = get_pokemon_form_by_name(conn, name, gender, threshold=100)
+        gendered_form = get_pokemon_form_by_name(db_controller, name, gender, threshold=100)
         if gendered_form:
             return gendered_form
 
@@ -96,68 +112,76 @@ def get_pokemon_form_by_name(conn, name: str, form: Optional[str] = None, thresh
 
     return None
 
-def process_evolution_chain(conn, evolution_chain, cache, gender: Optional[str] = None):
+def process_evolution_chain(db_controller, evolution_chain, cache, gender: Optional[str] = None):
     for stage in evolution_chain:
-        from_pfic = get_pokemon_form_by_name(conn, stage.pokemon, stage.form, gender=gender)
+        from_pfic = get_pokemon_form_by_name(db_controller, stage.pokemon, stage.form, gender=gender)
         if not from_pfic:
             logger.warning(f"Could not find PFIC for {stage.pokemon} {stage.form}")
             continue
 
         if stage.next_stage:
-            to_pfic = get_pokemon_form_by_name(conn, stage.next_stage.pokemon, stage.next_stage.form, gender=gender)
+            to_pfic = get_pokemon_form_by_name(db_controller, stage.next_stage.pokemon, stage.next_stage.form, gender=gender)
             if to_pfic:
                 evolution_info = EvolutionInfo(from_pfic, to_pfic, stage.next_stage.method)
-                insert_evolution_info(conn, evolution_info)
+                insert_evolution_info(db_controller, evolution_info)
 
                 if "breed" in stage.next_stage.method.lower():
-                    update_pokemon_baby_status(conn, from_pfic, True)
+                    update_pokemon_baby_status(db_controller, from_pfic, True)
 
         for branch in stage.branches:
-            to_pfic = get_pokemon_form_by_name(conn, branch.pokemon, branch.form, gender=gender)
+            to_pfic = get_pokemon_form_by_name(db_controller, branch.pokemon, branch.form, gender=gender)
             if to_pfic:
                 evolution_info = EvolutionInfo(from_pfic, to_pfic, branch.method)
-                insert_evolution_info(conn, evolution_info)
+                insert_evolution_info(db_controller, evolution_info)
 
                 if "breed" in branch.method.lower():
-                    update_pokemon_baby_status(conn, from_pfic, True)
+                    update_pokemon_baby_status(db_controller, from_pfic, True)
 
-def update_pokemon_baby_status(conn, from_pfic, is_baby_form):
-    cursor = conn.cursor()
-    cursor.execute('''
+def update_pokemon_baby_status(db_controller, from_pfic, is_baby_form):
+    db_controller.execute_query_with_commit('''
     UPDATE pokemon_forms
     SET is_baby_form = ?
     WHERE PFIC = ?
     ''', (is_baby_form, from_pfic))
-    conn.commit()
 
-def update_evolution_chains(cache, progress_callback=None):
-    conn = create_evolution_table()
 
-    cursor = conn.cursor()
-    cursor.execute('SELECT DISTINCT name, form_name FROM pokemon_forms')
-    pokemon_forms = cursor.fetchall()
-
-    for name, form in pokemon_forms:
-        if progress_callback:
-            progress_callback(f"Processing {name} {form if form else ''}")
-
-        if form and name in form:
-            form = form.replace(name, "").strip()
-
-        gender = None
-        if form and "male" in form.lower():
-            gender = form
-            form = None
-        
-        evolution_chain = get_evolution_data_from_bulbapedia(name, form, cache, gender)
-        if evolution_chain:
-            if name == "Tauros": # Bulbapedia has a weird formatting for Tauros.
-                for stage in evolution_chain:
-                    if stage.form:
-                        stage.form = stage.form.replace("Paldean Form(", "").replace(")", "").strip()
-            process_evolution_chain(conn, evolution_chain, cache, gender)
-
-    conn.close()
+def update_evolution_chains(cache, progress_callback=None):
+    db_controller = DBController('pokemon_forms.db', max_connections=20)
+    try:
+        db_controller.execute_query_with_commit('BEGIN')
+        pokemon_forms = db_controller.execute_query('''
+            SELECT pf.name, pf.form_name
+            FROM pokemon_forms pf
+            ORDER BY pf.national_dex, pf.form_name
+        ''')
+
+        for name, form in pokemon_forms:
+            if progress_callback:
+                progress_callback(f"Processing {name} {form if form else ''}")
+
+            if form and name in form:
+                form = form.replace(name, "").strip()
+
+            gender = None
+            if form and "male" in form.lower():
+                gender = form
+                form = None
+            
+            evolution_chain = get_evolution_data_from_bulbapedia(name, form, cache, gender)
+            if evolution_chain:
+                if name == "Tauros": # Bulbapedia has a weird formatting for Tauros.
+                    for stage in evolution_chain:
+                        if stage.form:
+                            stage.form = stage.form.replace("Paldean Form(", "").replace(")", "").strip()
+                process_evolution_chain(db_controller, evolution_chain, cache, gender)
+        db_controller.execute_query_with_commit('COMMIT')
+    except Exception as e:
+        # Rollback in case of error
+        db_controller.execute_query_with_commit('ROLLBACK')
+        logger.error(f"Error updating evolution chains: {str(e)}")
+        raise
+    finally:
+        db_controller.close()
 
 if __name__ == "__main__":
     cache = CacheManager()

DataGatherers/cache_manager.py

@@ -2,40 +2,102 @@ import sqlite3
 import json
 import time
 import requests
-from typing import Any, Optional
+from typing import Any, Optional, Dict, List
+import pickle
+from concurrent.futures import ThreadPoolExecutor
+from threading import Lock
 
 class CacheManager:
-    def __init__(self, db_name: str = 'pokemon_cache.db'):
-        self.conn = sqlite3.connect(db_name)
-        self.cursor = self.conn.cursor()
+    def __init__(self, db_name: str = 'pokemon_cache.db', max_connections: int = 10):
+        self.db_name = db_name
+        self.memory_cache: Dict[str, Any] = {}
+        self.expiry_disabled = True
+        self.lock = Lock()
+        self.connection_pool = ThreadPoolExecutor(max_workers=max_connections)
         self._create_cache_table()
+        self.fetch_url_lock = Lock()
+
+    def _create_connection(self):
+        return sqlite3.connect(self.db_name)
 
     def _create_cache_table(self):
-        self.cursor.execute('''
-            CREATE TABLE IF NOT EXISTS cache (
-                key TEXT PRIMARY KEY,
-                value TEXT,
-                timestamp FLOAT
-            )
-        ''')
-        self.conn.commit()
+        with self._create_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                CREATE TABLE IF NOT EXISTS cache (
+                    key TEXT PRIMARY KEY,
+                    value BLOB,
+                    timestamp FLOAT
+                )
+            ''')
+            conn.commit()
 
     def get(self, key: str) -> Optional[Any]:
-        self.cursor.execute('SELECT value, timestamp FROM cache WHERE key = ?', (key,))
-        result = self.cursor.fetchone()
+        # Check memory cache first
+        with self.lock:
+            if key in self.memory_cache:
+                return self.memory_cache[key]
+
+        # If not in memory, check database
+        def db_get():
+            with self._create_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute('SELECT value, timestamp FROM cache WHERE key = ?', (key,))
+                result = cursor.fetchone()
+                if result:
+                    value, timestamp = result
+                    return pickle.loads(value)
+            return None
+
+        result = self.connection_pool.submit(db_get).result()
         if result:
-            value, timestamp = result
-            return json.loads(value)
-        return None
+            with self.lock:
+                self.memory_cache[key] = result
+        return result
 
     def set(self, key: str, value: Any):
-        serialized_value = json.dumps(value)
+        serialized_value = pickle.dumps(value)
         timestamp = time.time()
-        self.cursor.execute('''
-            INSERT OR REPLACE INTO cache (key, value, timestamp)
-            VALUES (?, ?, ?)
-        ''', (key, serialized_value, timestamp))
-        self.conn.commit()
+
+        with self.lock:
+            self.memory_cache[key] = value
+
+        def db_set():
+            with self._create_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute('''
+                    INSERT OR REPLACE INTO cache (key, value, timestamp)
+                    VALUES (?, ?, ?)
+                ''', (key, serialized_value, timestamp))
+                conn.commit()
+
+        self.connection_pool.submit(db_set)
+
+    def bulk_get(self, keys: List[str]) -> Dict[str, Any]:
+        results = {}
+        db_keys = []
+
+        with self.lock:
+            for key in keys:
+                if key in self.memory_cache:
+                    results[key] = self.memory_cache[key]
+                else:
+                    db_keys.append(key)
+
+        if db_keys:
+            def db_bulk_get():
+                with self._create_connection() as conn:
+                    cursor = conn.cursor()
+                    placeholders = ','.join('?' for _ in db_keys)
+                    cursor.execute(f'SELECT key, value FROM cache WHERE key IN ({placeholders})', db_keys)
+                    return {key: pickle.loads(value) for key, value in cursor.fetchall()}
+
+            db_results = self.connection_pool.submit(db_bulk_get).result()
+            with self.lock:
+                self.memory_cache.update(db_results)
+                results.update(db_results)
+
+        return results
 
     def fetch_url(self, url: str, force_refresh: bool = False, expiry: int = 86400) -> Optional[str]:
         cache_key = f"url_{url}"
@@ -43,15 +105,16 @@ class CacheManager:
             cached_data = self.get(cache_key)
             if cached_data:
                 cached_time = cached_data['timestamp']
-                if time.time() - cached_time < expiry:
+                if time.time() - cached_time < expiry or self.expiry_disabled:
                     return cached_data['content']
 
+        #with self.fetch_url_lock:
         print(f"Fetching URL: {url}")
         response = requests.get(url)
         if response.status_code == 200:
             content = response.text
             self.set(cache_key, {
-                'content': content,
+            'content': content,
                 'timestamp': time.time()
             })
             time.sleep(1)
@@ -59,7 +122,7 @@ class CacheManager:
         return None
 
     def close(self):
-        self.conn.close()
+        self.connection_pool.shutdown()
 
 # Usage example
 if __name__ == "__main__":

DataGatherers/pokemondb_scraper.py

@@ -6,6 +6,8 @@ import os
 import sqlite3
 import sys
 import logging
+import re
+import unicodedata
 
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
@@ -122,6 +124,32 @@ def get_pokemon_sprites_page_data(cache: CacheManager, pokemon_name: str):
     url = f"https://pokemondb.net/sprites/{pokemon_name}"
     return cache.fetch_url(url)
 
+def get_pokemon_dex_page(cache: CacheManager, pokemon_name: str):
+    url = f"https://pokemondb.net/pokedex/{pokemon_name}"
+    return cache.fetch_url(url)
+
+def remove_accents(input_str):
+    nfkd_form = unicodedata.normalize('NFKD', input_str)
+    return u"".join([c for c in nfkd_form if not unicodedata.combining(c)])
+
+def compare_forms(a, b):
+    if a == None or b == None:
+        return False
+    
+    if a == b:
+        return True
+    
+    temp_a = a.lower().replace("forme", "").replace("form", "").replace("é", "e").strip()
+    temp_b = b.lower().replace("forme", "").replace("form", "").replace("é", "e").strip()
+
+    temp_a = temp_a.replace("deputante", "debutante").replace("p'au", "pa'u").replace("blood moon", "bloodmoon")
+    temp_b = temp_b.replace("deputante", "debutante").replace("p'au", "pa'u").replace("blood moon", "bloodmoon")
+
+    if temp_a == temp_b:
+        return True
+
+    return False
+
 def download_image(url, filename):
     response = requests.get(url)
     if response.status_code == 200:
@@ -205,18 +233,73 @@ def retrieve_all_pokemon_forms(cache: CacheManager, progress_callback=None):
 
                     form_name = "None"
                     if sprite.find('small'):
-                        form_name = sprite.find('small').get_text(strip=True)
+                        smalls = sprite.find_all('small')
+                        form_name = ""
+                        for small in smalls:
+                            form_name += small.get_text(strip=True) + " "
+                        form_name = form_name.strip()
                     logger.info(f'{sprite_url}, {form_name}')
                     if form_name != "None":
                         form += 1
                     gender = 0
-                    if form_name == "Female":
+                    if form_name.startswith("Male"):
                         form -= 1
                         gender = 1
-                    elif form_name == "Male":
+                    elif form_name.startswith("Female"):
                         form -= 1
                         gender = 2
 
+                    dex_page_data = get_pokemon_dex_page(cache, pokemon_name.replace("'", "").replace(".", "-").replace(" ", ""))
+                    if dex_page_data:
+                        dex_soup = BeautifulSoup(dex_page_data, 'html.parser')
+                    
+                        #Find a heading that has the pokemon name in it
+                        dex_header = dex_soup.find('h1', string=pokemon_name)
+                        if dex_header:
+                            #The next <p> tag contains the generation number, in the format "{pokemon name} is a {type}(/{2nd_type}) type Pokémon introduced in Generation {generation number}."
+                            generation_tag = dex_header.find_next('p')
+                            dex_text = generation_tag.get_text()
+                            pattern = r'^(.+?) is a (\w+)(?:/(\w+))? type Pokémon introduced in Generation (\d+)\.$'
+                            match = re.match(pattern, dex_text)
+                            if match:
+                                name, type1, type2, gen = match.groups()
+                                generation = int(gen)
+
+                            if form_name != "None":
+                                next_tag = generation_tag.find_next('p')
+                                if next_tag:
+                                    extra_text = next_tag.get_text()
+                                    extra_text = remove_accents(extra_text)
+                                    form_pattern = r'a(?:n)? (\w+) Form(?:,)? introduced in (?:the )?([\w\s:]+)(?:\/([\w\s:]+))?'
+                                    update_pattern = r'a(?:n)? (\w+) form(?:,)? available in the latest update to ([\w\s:]+)(?:& ([\w\s:]+))?'
+                                    multiple_forms_pattern = r'has (?:\w+) new (\w+) Form(?:s)?(?:,)? available in (?:the )?([\w\s:]+)(?:& ([\w\s:]+))?'
+                                    expansion_pass_pattern = r'a(?:n)? (\w+) form(?:,)? introduced in the Crown Tundra Expansion Pass to ([\w\s:]+)(?:& ([\w\s:]+))?'
+                                    patterns = [form_pattern, update_pattern, multiple_forms_pattern, expansion_pass_pattern]
+                                    test_form = form_name.replace(pokemon_name, "").replace("Male", "").replace("Female", "").strip()
+                                    if pokemon_name == "Tauros" and (form_name == "Aqua Breed" or form_name == "Blaze Breed" or form_name == "Combat Breed"):
+                                        test_form = "Paldean"
+                                    for pattern in patterns:
+                                        matches = re.findall(pattern, extra_text, re.IGNORECASE)
+                                        generation_found = False
+                                        for i, (regional, game1, game2) in enumerate(matches, 1):
+                                            if compare_forms(test_form, regional):
+                                                target_game = game1.replace("Pokemon", "").strip()
+                                                cursor = pokemon_db_conn.cursor()
+                                                cursor.execute('''
+                                                    SELECT g.generation
+                                                    FROM games g
+                                                    LEFT JOIN alternate_game_names agn ON g.id = agn.game_id
+                                                    WHERE g.name = ? OR agn.alternate_name = ?
+                                                    LIMIT 1
+                                                ''', (target_game, target_game))
+                                                result = cursor.fetchone()
+                                                if result:
+                                                    generation = result[0]
+                                                    generation_found = True
+                                                    break
+                                        if generation_found:
+                                            break
+
                     pokemon_form = PokemonForm(
                         id=format_pokemon_id(national_dex_index, generation, form, gender),
                         name=pokemon_name,

DataGatherers/update_location_information.py

@@ -12,10 +12,14 @@ from bs4 import BeautifulSoup, Tag
 import re
 import time
 import unicodedata
+import concurrent.futures
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 import logging
 logger = logging.getLogger('ui_feedback')
 
+from db_controller import DBController  # Import your DBController
+
 def create_encounters_table():
     conn = sqlite3.connect('pokemon_forms.db')
     cursor = conn.cursor()
@@ -227,9 +231,7 @@ def build_query_string(table_name, criteria):
 
     return query + " AND ".join(conditions), values
     
-def save_encounter(conn, pfic, game, location, days, times, dual_slot, static_encounter, static_encounter_count, extra_text, stars, rods, fishing, starter):
-    cursor = conn.cursor()
-
+def save_encounter(db_controller, pfic, game, location, days, times, dual_slot, static_encounter, static_encounter_count, extra_text, stars, rods, fishing, starter):
     extra_text_str = ' '.join(extra_text) if extra_text else None
     stars_str = ','.join(sorted(stars)) if stars else None
     rods_str = ','.join(sorted(rods)) if rods else None
@@ -264,12 +266,10 @@ def save_encounter(conn, pfic, game, location, days, times, dual_slot, static_en
             criteria["time"] = None
             query, values = build_query_string("encounters", criteria)
             # Check if an identical record already exists
-            cursor.execute(query, values)
-
-            encounter = cursor.fetchone()
+            result = db_controller.execute_query(query, values)
 
-            if encounter == None or encounter[0] == 0:
-                cursor.execute(insert_query, (pfic, game_id, location, day, None, dual_slot, static_encounter_count, 
+            if not result:
+                db_controller.execute_query_with_commit(insert_query, (pfic, game_id, location, day, None, dual_slot, static_encounter_count, 
                       static_encounter, extra_text_str, stars_str, rods_str, fishing, starter))
                 logger.info(f"New encounter added for {pfic} in {game} at {location} on {day}")
             else:
@@ -281,12 +281,10 @@ def save_encounter(conn, pfic, game, location, days, times, dual_slot, static_en
             criteria["time"] = time
             query, values = build_query_string("encounters", criteria)
             # Check if an identical record already exists
-            cursor.execute(query, values)
+            result = db_controller.execute_query(query, values)
 
-            encounter = cursor.fetchone()
-
-            if encounter == None or encounter[0] == 0:
-                cursor.execute(insert_query, (pfic, game_id, location, None, time, dual_slot, static_encounter_count, 
+            if not result:
+                db_controller.execute_query_with_commit(insert_query, (pfic, game_id, location, None, time, dual_slot, static_encounter_count, 
                       static_encounter, extra_text_str, stars_str, rods_str, fishing, starter))
                 logger.info(f"New encounter added for {pfic} in {game} at {location} at {time}")
             else:
@@ -297,18 +295,71 @@ def save_encounter(conn, pfic, game, location, days, times, dual_slot, static_en
         criteria["time"] = None
         query, values = build_query_string("encounters", criteria)
         # Check if an identical record already exists
-        cursor.execute(query, values)
-
-        encounter = cursor.fetchone()
+        result = db_controller.execute_query(query, values)
 
-        if encounter == None or encounter[0] == 0:
-            cursor.execute(insert_query, (pfic, game_id, location, None, None, dual_slot, static_encounter_count, 
+        if not result:
+            db_controller.execute_query_with_commit(insert_query, (pfic, game_id, location, None, None, dual_slot, static_encounter_count, 
                   static_encounter, extra_text_str, stars_str, rods_str, fishing, starter))
             logger.info(f"New encounter added for {pfic} in {game} at {location}")
         else:
             logger.info(f"Identical encounter already exists for {pfic} in {game} at {location}")
 
-    conn.commit()
+def save_evolve_encounter(db_controller, pfic, game, days, times, from_pfic):
+    game_id = event_system.call_sync('get_game_id_for_name', game)
+
+    insert_query = '''
+    INSERT INTO evolve_encounters 
+    (pfic, game_id, day, time, from_pfic)
+    VALUES (?, ?, ?, ?, ?)
+    '''
+
+    criteria = {
+        "pfic": pfic,
+        "game_id": game_id,
+        "day": None,
+        "time": None,
+        "from_pfic": from_pfic
+    }
+
+    if len(days) > 0:
+        for day in days:
+            criteria["day"] = day
+            criteria["time"] = None
+            query, values = build_query_string("evolve_encounters", criteria)
+            # Check if an identical record already exists
+            result = db_controller.execute_query(query, values)
+
+            if not result:
+                db_controller.execute_query_with_commit(insert_query, (pfic, game_id, day, None, from_pfic))
+                logger.info(f"New evolve encounter added for {pfic} in {game} on {day}")
+            else:
+                logger.info(f"Identical evolve encounter already exists for {pfic} in {game} on {day}")
+
+    elif len(times) > 0:
+        for time in times:
+            criteria["day"] = None
+            criteria["time"] = time
+            query, values = build_query_string("evolve_encounters", criteria)
+            # Check if an identical record already exists
+            result = db_controller.execute_query(query, values)
+            
+            if not result:
+                db_controller.execute_query_with_commit(insert_query, (pfic, game_id, None, time, from_pfic))
+                logger.info(f"New evolve encounter added for {pfic} in {game} at {time}")
+            else:
+                logger.info(f"Identical evolve encounter already exists for {pfic} in {game} at {time}")
+    else:
+        criteria["day"] = None
+        criteria["time"] = None
+        query, values = build_query_string("evolve_encounters", criteria)
+        # Check if an identical record already exists
+        result = db_controller.execute_query(query, values)
+
+        if not result:
+            db_controller.execute_query_with_commit(insert_query, (pfic, game_id, None, None, from_pfic))     
+            logger.info(f"New evolve encounter added for {pfic} in {game}")
+        else:
+            logger.info(f"Identical evolve encounter already exists for {pfic} in {game}")
 
 def compare_forms(a, b):
     if a == b:
@@ -325,7 +376,7 @@ def compare_forms(a, b):
 
     return False
 
-def process_pokemon_for_location_data(pfic, name, form, national_dex, default_forms, cache, conn):
+def process_pokemon_for_location_data(pfic, name, form, national_dex, default_forms, cache, db_controller):
     logger.info(f"Processing {name} {form if form else ''}")
 
     if form and name in form:
@@ -349,7 +400,13 @@ def process_pokemon_for_location_data(pfic, name, form, national_dex, default_fo
     if name.lower() == "ho-oh":
         name = "Ho-Oh"
 
-    if form and form.lower() == "female":
+    if form and form.startswith("Female"):
+        form = form.replace("Female", "").strip()
+
+    if form and form.startswith("Male"):
+        form = form.replace("Male", "").strip()
+
+    if form == "":
         form = None
 
     search_form = form
@@ -358,7 +415,6 @@ def process_pokemon_for_location_data(pfic, name, form, national_dex, default_fo
         "trade",
         "time capsule",
         "unobtainable",
-        "evolve",
         "tradeversion",
         "poké transfer",
         "friend safari",
@@ -402,36 +458,76 @@ def process_pokemon_for_location_data(pfic, name, form, national_dex, default_fo
                 logger.info(f"Found in {encounter}:")
                 print_encounter = False
 
-            remaining, details = extract_additional_information(location["tag"])
-            routes, remaining = extract_routes(remaining)
-            logger.info(f"Routes: {routes}")
-            logger.info(f"Remaining: {remaining.strip()}")
-            logger.info(f"Details: {details}")
+            if "evolve" in test_location:
+                remaining, details = extract_additional_information(location["tag"])
+                evolve_info = extract_evolve_information(remaining, db_controller)
+
+                if evolve_info:
+                    logger.info(f"Evolve Info: {evolve_info}")
+                    save_evolve_encounter(db_controller, pfic, encounter, details["days"], details["times"], evolve_info["evolve_from"])
+            else:    
+                remaining, details = extract_additional_information(location["tag"])
+                routes, remaining = extract_routes(remaining)
+                logger.info(f"Routes: {routes}")
+                logger.info(f"Remaining: {remaining.strip()}")
+                logger.info(f"Details: {details}")
+
+                if len(details["times"]) > 0:
+                    logger.info("Stupid Data")
+
+                for route in routes:
+                    route_name = f"Route {route}"
+                    save_encounter(db_controller, pfic, encounter, route_name, details["days"], details["times"], details["dual_slot"], details["static_encounter"], details["static_encounter_count"], details["extra_text"], details["stars"], details["Rods"], details["Fishing"], details["starter"] )
 
-            if len(details["times"]) > 0:
-                logger.info("Stupid Data")
+                if remaining != "":
+                    remaining_locations = remaining.replace(" and ", ",").split(",")
+                    for remaining_location in remaining_locations:
+                        if remaining_location.strip() == "":
+                            continue
 
-            for route in routes:
-                route_name = f"Route {route}"
-                save_encounter(conn, pfic, encounter, route_name, details["days"], details["times"], details["dual_slot"], details["static_encounter"], details["static_encounter_count"], details["extra_text"], details["stars"], details["Rods"], details["Fishing"], details["starter"] )
+                        save_encounter(db_controller, pfic, encounter, remaining_location.strip(), details["days"], details["times"], details["dual_slot"], details["static_encounter"], details["static_encounter_count"], details["extra_text"], details["stars"], details["Rods"], details["Fishing"], details["starter"]  )
 
-            if remaining != "":
-                remaining_locations = remaining.replace(" and ", ",").split(",")
-                for remaining_location in remaining_locations:
-                    if remaining_location.strip() == "":
-                        continue
+def extract_evolve_information(s: str, db_controller):
+    details = {}
+    if s is None or s == "":
+        return details
+    
+    s = s.replace("Evolve", "")
+    
+    parts = s.split(" ")
+
+    if len(parts) >= 1:
+        target_pokemon = parts[0].strip()
+
+        form = None
+        if "♀" in target_pokemon:
+            target_pokemon = target_pokemon.replace("♀", "").strip()
+            form = "Female"
+
+        if "♂" in target_pokemon:
+            target_pokemon = target_pokemon.replace("♂", "").strip()
+            form = None
+
+        result = db_controller.execute_query('''
+            SELECT e.from_pfic
+            FROM evolution_chains e
+            JOIN pokemon_forms pf ON pf.PFIC = e.from_pfic
+            WHERE pf.name = ?
+        ''', (target_pokemon,))
 
-                    save_encounter(conn, pfic, encounter, remaining_location.strip(), details["days"], details["times"], details["dual_slot"], details["static_encounter"], details["static_encounter_count"], details["extra_text"], details["stars"], details["Rods"], details["Fishing"], details["starter"]  )
+        if result:
+            details["evolve_from"] = result[0][0]
+
+    return details
 
 def update_location_information(cache, progress_callback=None):
-    conn = create_encounters_table()
-    cursor = conn.cursor()
-    cursor.execute('''
-            SELECT pf.PFIC, pf.name, pf.form_name, pf.national_dex
-            FROM pokemon_forms pf
-            ORDER BY pf.national_dex, pf.form_name
-        ''')
-    pokemon_forms = cursor.fetchall()
+    db_controller = DBController('pokemon_forms.db', max_connections=20)  # Adjust max_connections as needed
+    
+    pokemon_forms = db_controller.execute_query('''
+        SELECT pf.PFIC, pf.name, pf.form_name, pf.national_dex
+        FROM pokemon_forms pf
+        ORDER BY pf.national_dex, pf.form_name
+    ''')
 
     try:
         with open('./DataGatherers/DefaultForms.json', 'r') as f:
@@ -439,14 +535,27 @@ def update_location_information(cache, progress_callback=None):
     except FileNotFoundError:
         default_forms = []
 
-    for pfic, name, form, national_dex in pokemon_forms:
-        if progress_callback:
-            progress_callback(f"Processing {name} {form if form else ''}")
-        process_pokemon_for_location_data(pfic, name, form, national_dex, default_forms, cache, conn)
+    def process_single_pokemon(args):
+        pfic, name, form, national_dex = args
+        try:
+            process_pokemon_for_location_data(pfic, name, form, national_dex, default_forms, cache, db_controller)
+            if progress_callback:
+                progress_callback(f"Processed {name} {form if form else ''}")
+        except Exception as exc:
+            logger.error(f'Error processing {name} {form}: {exc}')
+
+    with ThreadPoolExecutor(max_workers=10) as executor:
+        futures = [executor.submit(process_single_pokemon, form_data) for form_data in pokemon_forms]
+        
+        for future in as_completed(futures):
+            try:
+                future.result()
+            except Exception as exc:
+                logger.error(f'A task generated an exception: {exc}')
 
-    conn.close()
+    db_controller.close()
 
-if __name__ == "__main__":
-    cache = CacheManager()
-    update_location_information(cache)
+import cProfile
 
+def profile_update_location_information(cache, progress_callback=None):
+    cProfile.runctx('update_location_information(cache, progress_callback)', globals(), locals(), 'profile_stats')

Site/OriginDex.py

@@ -18,97 +18,13 @@ def load_pokemon_data():
             ps.storable_in_home
         FROM pokemon_forms pf
         JOIN pokemon_storage ps ON pf.PFIC = ps.PFIC
+
         WHERE ps.storable_in_home = 1
         ORDER BY pf.PFIC
     ''')
     
     pokemon_data = cursor.fetchall()
 
-    # Function to get the origin mark for a Pokémon
-    def get_origin_mark(pfic, generation):
-        # Step 1: Check for previous evolution in the same generation
-        cursor.execute('''
-            WITH RECURSIVE EvolutionChain AS (
-                SELECT ec.to_pfic, ec.from_pfic, pf_to.generation as to_gen, pf_from.generation as from_gen
-                FROM evolution_chains ec
-                JOIN pokemon_forms pf_to ON ec.to_pfic = pf_to.PFIC
-                JOIN pokemon_forms pf_from ON ec.from_pfic = pf_from.PFIC
-                WHERE ec.to_pfic = ? AND pf_to.generation = ?
-                UNION ALL
-                SELECT ec.to_pfic, ec.from_pfic, pf_to.generation, pf_from.generation
-                FROM evolution_chains ec
-                JOIN EvolutionChain e ON ec.to_pfic = e.from_pfic
-                JOIN pokemon_forms pf_to ON ec.to_pfic = pf_to.PFIC
-                JOIN pokemon_forms pf_from ON ec.from_pfic = pf_from.PFIC
-                WHERE pf_to.generation = ? AND pf_from.generation = ?
-            )
-            SELECT from_pfic
-            FROM EvolutionChain
-            WHERE from_pfic IS NOT NULL AND from_gen = ?
-            ORDER BY from_pfic
-            LIMIT 1
-        ''', (pfic, generation, generation, generation, generation))
-        base_pfic = cursor.fetchone()
-        
-        if base_pfic:
-            pfic = base_pfic[0]
-        
-        # Step 2: Look for the earliest encounter for this form in the current generation
-        cursor.execute('''
-            SELECT g.name, m.icon_path, m.id
-            FROM encounters e
-            JOIN games g ON e.game_id = g.id
-            LEFT JOIN mark_game_associations mga ON g.id = mga.game_id
-            LEFT JOIN marks m ON mga.mark_id = m.id
-            WHERE e.pfic = ? AND g.generation = ?
-            ORDER BY g.id
-            LIMIT 1
-        ''', (pfic, generation))
-        encounter_data = cursor.fetchone()
-        
-        if encounter_data:
-            return encounter_data
-        
-        # Step 3: Look for encounters of previous evolutions in the current generation
-        cursor.execute('''
-            WITH RECURSIVE EvolutionChain AS (
-                SELECT ec.to_pfic, ec.from_pfic
-                FROM evolution_chains ec
-                WHERE ec.to_pfic = ?
-                UNION ALL
-                SELECT ec.to_pfic, ec.from_pfic
-                FROM evolution_chains ec
-                JOIN EvolutionChain e ON ec.to_pfic = e.from_pfic
-            )
-            SELECT g.name, m.icon_path, m.id
-            FROM EvolutionChain ec
-            JOIN encounters e ON ec.from_pfic = e.pfic
-            JOIN games g ON e.game_id = g.id
-            LEFT JOIN mark_game_associations mga ON g.id = mga.game_id
-            LEFT JOIN marks m ON mga.mark_id = m.id
-            WHERE g.generation = ?
-            ORDER BY g.id
-            LIMIT 1
-        ''', (pfic, generation))
-        evolution_encounter_data = cursor.fetchone()
-        
-        if evolution_encounter_data:
-            return evolution_encounter_data
-        
-        # Step 4: Use the mark of the earliest game of the generation
-        cursor.execute('''
-            SELECT g.name, m.icon_path, m.id
-            FROM games g
-            LEFT JOIN mark_game_associations mga ON g.id = mga.game_id
-            LEFT JOIN marks m ON mga.mark_id = m.id
-            WHERE g.generation = ?
-            ORDER BY g.id
-            LIMIT 1
-        ''', (generation,))
-        generation_data = cursor.fetchone()
-        
-        return generation_data if generation_data else (None, None, None)
-
     # Process the data
     current_group = []
     current_generation = None
@@ -118,7 +34,25 @@ def load_pokemon_data():
     for row in pokemon_data:
         national_dex, name, form_name, pfic, generation, storable_in_home = row
 
-        earliest_game, mark_icon, mark_id = get_origin_mark(pfic, generation)
+        mark_id = None
+        mark_icon = None
+        
+        cursor.execute('''
+            SELECT mark_id
+            FROM form_marks
+            WHERE pfic = ?
+        ''', (pfic,))
+        result = cursor.fetchone()
+        mark_id = result[0] if result else None
+
+        if mark_id:
+            cursor.execute('''
+                SELECT icon_path
+                FROM marks
+                WHERE id = ?
+            ''', (mark_id,))
+            result = cursor.fetchone()
+            mark_icon = result[0] if result else None
 
         pokemon = {
             'pfic': pfic,
@@ -128,7 +62,6 @@ def load_pokemon_data():
             'Image': f"images/pokemon/{pfic}.png",
             'IsDefault': form_name is None or form_name == '',
             'Generation': generation,
-            'EarliestGame': earliest_game,
             'StorableInHome': storable_in_home,
             'MarkIcon': mark_icon,
             'MarkID': mark_id