From 41c9e254e5c825f19bdb143f961b8e207d92ef83 Mon Sep 17 00:00:00 2001 From: Dan Date: Mon, 4 Nov 2024 16:55:26 +0000 Subject: [PATCH 01/21] - Initial database for pokemon_forms is in and sorted. - Updated the list view to auto populate on boot --- database/__init__.py | 0 database/db_controller.py | 98 ++++++++++++++++++++++++++++++++++++ db.py | 2 + main.py | 4 +- ui/main_window_controller.py | 11 +++- ui/main_window_view.py | 12 ++--- utility/functions.py | 4 ++ 7 files changed, 120 insertions(+), 11 deletions(-) create mode 100644 database/__init__.py create mode 100644 database/db_controller.py create mode 100644 db.py diff --git a/database/__init__.py b/database/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/database/db_controller.py b/database/db_controller.py new file mode 100644 index 0000000..08e4ba1 --- /dev/null +++ b/database/db_controller.py @@ -0,0 +1,98 @@ +import sqlite3 +import threading +import json + +class DBController: + def __init__(self, db_path=':memory:', max_connections=10): + self.db_path = db_path + self.lock = threading.Lock() + self.conn = sqlite3.connect(db_path, check_same_thread=False) + self.conn.row_factory = sqlite3.Row + self.cursor = self.conn.cursor() + self.init_database() + + def init_database(self): + disk_conn = sqlite3.connect('pokemon_forms.db') + disk_cursor = disk_conn.cursor() + + # Create tables in the file-based database + self.create_pokemon_forms_table(disk_cursor) + + # Commit changes to the file-based database + disk_conn.commit() + + # Copy the file-based database to the in-memory database + disk_conn.backup(self.conn) + + # Close the file-based database connection + disk_conn.close() + + def save_changes(self): + with self.lock: + # Count the number of records before backup for verification + self.cursor.execute('SELECT COUNT(*) FROM pokemon_forms') + count = self.cursor.fetchone()[0] + print(f"Records in memory before backup: {count}") + + # Back up the master connection to disk + disk_conn = sqlite3.connect('pokemon_forms.db') + with disk_conn: + self.conn.backup(disk_conn) + disk_conn.close() + + def close(self): + self.save_changes() + self.conn.close() + + def create_pokemon_forms_table(self, cursor): + cursor.execute(''' + CREATE TABLE IF NOT EXISTS pokemon_forms ( + PFIC TEXT PRIMARY KEY, + data JSON NOT NULL + ) + ''') + + def add_pokemon_form(self, pfic, name, form_name, national_dex, generation, sprite_url): + data = { + "name": name, + "form_name": form_name, + "national_dex": national_dex, + "generation": generation, + "sprite_url": sprite_url, + "is_baby_form": False, + "storable_in_home": True, # Example additional field + } + + with self.lock: + self.cursor.execute(''' + INSERT OR REPLACE INTO pokemon_forms (PFIC, data) VALUES (?, ?) + ''', (pfic, json.dumps(data))) + self.conn.commit() + print(f"Added: {pfic}, {name}") + + def get_pokemon_details(self, pfic): + self.cursor.execute(''' + SELECT JSON_EXTRACT(data, '$.name') AS name, + JSON_EXTRACT(data, '$.form_name') AS form_name, + JSON_EXTRACT(data, '$.national_dex') AS national_dex, + JSON_EXTRACT(data, '$.generation') AS generation, + JSON_EXTRACT(data, '$.is_baby_form') AS is_baby_form, + FROM pokemon_forms + WHERE PFIC = ? + ''', (pfic,)) + results = self.cursor.fetchone() + return dict(results) + + def get_list_of_pokemon_forms(self): + self.cursor.execute(''' + SELECT JSON_EXTRACT(data, '$.name') AS name, + JSON_EXTRACT(data, '$.form_name') AS form_name, + JSON_EXTRACT(data, '$.national_dex') AS national_dex, + JSON_EXTRACT(data, '$.generation') AS generation, + JSON_EXTRACT(data, '$.is_baby_form') AS is_baby_form, + PFIC as pfic + FROM pokemon_forms + ''',) + results = self.cursor.fetchall() + + return [dict(row) for row in results] diff --git a/db.py b/db.py new file mode 100644 index 0000000..17dda80 --- /dev/null +++ b/db.py @@ -0,0 +1,2 @@ +from database.db_controller import DBController +db = DBController() \ No newline at end of file diff --git a/main.py b/main.py index 7dd806a..7eb5ed4 100644 --- a/main.py +++ b/main.py @@ -7,6 +7,7 @@ from PyQt6 import QtWidgets from ui.main_window_view import PokemonUI from cache import cache +from db import db def main(): import sys @@ -20,4 +21,5 @@ if __name__ == "__main__": main() finally: # Ensure the cache is closed at the end of the application - cache.close() \ No newline at end of file + cache.close() + db.close() \ No newline at end of file diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index 30e1919..90b301e 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -4,6 +4,8 @@ from PyQt6.QtGui import QAction from ui.workers import GatherPokemonFormsWorker +from db import db + class MainWindowController: def __init__(self, view): self.view = view @@ -16,7 +18,7 @@ class MainWindowController: def initialize_pokemon_list(self, data): self.pokemon_data_cache = data - self.view.update_pokemon_list(data) + self.view.update_pokemon_forms(data) def filter_pokemon_list(self): self.filter_timer.start() @@ -48,7 +50,7 @@ class MainWindowController: filtered_data.append((pfic, display_name)) # Update the view with the filtered data - self.view.update_pokemon_list(filtered_data) + self.view.update_pokemon_forms(filtered_data) def show_pokemon_context_menu(self, position): item = self.view.pokemon_list.itemAt(position) @@ -86,8 +88,13 @@ class MainWindowController: def on_forms_gathered(self, data): # This method will be called in the main thread when the worker finishes # Update the UI with the gathered forms + for pokemon in data: + db.add_pokemon_form(pokemon["pfic"], pokemon["name"], pokemon["form_name"], pokemon["national_dex"], pokemon["generation"], pokemon["sprite_url"]) + self.pokemon_data_cache = data self.view.update_pokemon_forms(data) + db.save_changes() + def gather_home_storage_info(self): pass diff --git a/ui/main_window_view.py b/ui/main_window_view.py index b910d47..5862b6f 100644 --- a/ui/main_window_view.py +++ b/ui/main_window_view.py @@ -6,11 +6,15 @@ from PyQt6.QtCore import Qt, QSize, QTimer, QMetaObject from PyQt6.QtGui import QPixmap, QFontMetrics, QColor, QAction from .main_window_controller import MainWindowController +from db import db + class PokemonUI(QWidget): def __init__(self, parent=None): super().__init__(parent) self.controller = MainWindowController(self) self.setup_ui() + data = db.get_list_of_pokemon_forms() + self.controller.initialize_pokemon_list(data) def setup_ui(self): main_layout = QVBoxLayout(self) @@ -209,14 +213,6 @@ class PokemonUI(QWidget): #self.load_exclusive_sets() - def update_pokemon_list(self, data): - self.pokemon_list.clear() - - for pfic, display_name in data: - item = QListWidgetItem(display_name) - item.setData(Qt.ItemDataRole.UserRole, pfic) - self.pokemon_list.addItem(item) - def update_pokemon_forms(self, data): self.pokemon_list.clear() diff --git a/utility/functions.py b/utility/functions.py index b368391..d917bb4 100644 --- a/utility/functions.py +++ b/utility/functions.py @@ -4,6 +4,10 @@ import unicodedata def format_pokemon_id(national_dex: int, region_code: int, form_index: int, gender_code: int) -> str: return f"{national_dex:04d}-{region_code:02d}-{form_index:03d}-{gender_code}" +def parse_pfic(pfic): + parts = pfic.split('-') + return tuple(int(part) if part.isdigit() else part for part in parts) + def compare_pokemon_forms(a, b): if a == None or b == None: return False -- 2.30.2 From 2946c95109953b517588e58cb0936442e0620a53 Mon Sep 17 00:00:00 2001 From: Quildra Date: Mon, 4 Nov 2024 21:40:54 +0000 Subject: [PATCH 02/21] - Hook up the on select for the details pane --- database/db_controller.py | 10 ++++++---- ui/main_window_controller.py | 27 ++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/database/db_controller.py b/database/db_controller.py index 08e4ba1..5b58741 100644 --- a/database/db_controller.py +++ b/database/db_controller.py @@ -77,6 +77,7 @@ class DBController: JSON_EXTRACT(data, '$.national_dex') AS national_dex, JSON_EXTRACT(data, '$.generation') AS generation, JSON_EXTRACT(data, '$.is_baby_form') AS is_baby_form, + JSON_EXTRACT(data, '$.storable_in_home') AS storable_in_home FROM pokemon_forms WHERE PFIC = ? ''', (pfic,)) @@ -86,10 +87,11 @@ class DBController: def get_list_of_pokemon_forms(self): self.cursor.execute(''' SELECT JSON_EXTRACT(data, '$.name') AS name, - JSON_EXTRACT(data, '$.form_name') AS form_name, - JSON_EXTRACT(data, '$.national_dex') AS national_dex, - JSON_EXTRACT(data, '$.generation') AS generation, - JSON_EXTRACT(data, '$.is_baby_form') AS is_baby_form, + JSON_EXTRACT(data, '$.form_name') AS form_name, + JSON_EXTRACT(data, '$.national_dex') AS national_dex, + JSON_EXTRACT(data, '$.generation') AS generation, + JSON_EXTRACT(data, '$.is_baby_form') AS is_baby_form, + JSON_EXTRACT(data, '$.storable_in_home') AS storable_in_home, PFIC as pfic FROM pokemon_forms ''',) diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index 90b301e..ac0b0bf 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -1,6 +1,7 @@ from PyQt6.QtCore import Qt, QTimer, QThreadPool from PyQt6.QtWidgets import QMenu from PyQt6.QtGui import QAction +import os from ui.workers import GatherPokemonFormsWorker @@ -123,4 +124,28 @@ class MainWindowController: pass def add_encounter_to_set(self): - pass \ No newline at end of file + pass + + def refresh_pokemon_details_panel(self, pfic): + details = db.get_pokemon_details(pfic) + if details: + self.view.name_label.setText(details["name"]) + self.view.form_name_label.setText(details["form_name"] if details["form_name"] else "") + self.view.national_dex_label.setText(str(details["national_dex"])) + self.view.generation_label.setText(str(details["generation"])) + self.view.home_checkbox.setChecked(bool(details["storable_in_home"])) + #self.view.home_checkbox.stateChanged.connect(self.update_home_storable) + self.view.is_baby_form_checkbox.setChecked(bool(details["is_baby_form"])) + + image_path = f"images-new/{pfic}.png" + if os.path.exists(image_path): + pixmap = QPixmap(image_path) + self.view.image_label.setPixmap(pixmap.scaled(150, 150, Qt.AspectRatioMode.KeepAspectRatio, Qt.TransformationMode.SmoothTransformation)) + else: + self.view.image_label.setText("Image not found") + + #self.load_evolution_chain(pfic) + #self.load_encounter_locations(pfic) + self.current_pfic = pfic + + -- 2.30.2 From e08b5b86e15a222c5298ecb6244c367557a358bf Mon Sep 17 00:00:00 2001 From: Dan Date: Tue, 5 Nov 2024 08:16:41 +0000 Subject: [PATCH 03/21] - Update to get home storage working as before --- database/db_controller.py | 25 ++- ui/main_window_controller.py | 12 +- .../gather_home_storage_status_worker.py | 148 ++++++++++++++++++ .../gather_pokemon_forms_worker.py} | 0 utility/data.py | 55 +++++++ utility/functions.py | 28 +++- 6 files changed, 265 insertions(+), 3 deletions(-) create mode 100644 ui/workers/gather_home_storage_status_worker.py rename ui/{workers.py => workers/gather_pokemon_forms_worker.py} (100%) diff --git a/database/db_controller.py b/database/db_controller.py index 5b58741..2872c73 100644 --- a/database/db_controller.py +++ b/database/db_controller.py @@ -60,7 +60,7 @@ class DBController: "generation": generation, "sprite_url": sprite_url, "is_baby_form": False, - "storable_in_home": True, # Example additional field + "storable_in_home": False, } with self.lock: @@ -98,3 +98,26 @@ class DBController: results = self.cursor.fetchall() return [dict(row) for row in results] + + def update_home_status(self, pfic, status): + self.update_pokemon_field(pfic, "storable_in_home", status) + pass + + def update_pokemon_field(self, pfic, field_name, new_value): + # Fetch the existing record + self.cursor.execute('SELECT data FROM pokemon_forms WHERE PFIC = ?', (pfic,)) + result = self.cursor.fetchone() + + if result: + # Load the JSON data and update the field + data = json.loads(result[0]) + data[field_name] = new_value + + # Update the record with the modified JSON + updated_data_str = json.dumps(data) + self.cursor.execute(''' + UPDATE pokemon_forms + SET data = ? + WHERE PFIC = ? + ''', (updated_data_str, pfic)) + self.conn.commit() diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index ac0b0bf..7e20b75 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -3,7 +3,8 @@ from PyQt6.QtWidgets import QMenu from PyQt6.QtGui import QAction import os -from ui.workers import GatherPokemonFormsWorker +from ui.workers.gather_home_storage_status_worker import GatherHomeStorageStatus +from ui.workers.gather_pokemon_forms_worker import GatherPokemonFormsWorker from db import db @@ -97,6 +98,15 @@ class MainWindowController: db.save_changes() def gather_home_storage_info(self): + worker = GatherHomeStorageStatus() + worker.signals.finished.connect(self.on_home_status_gathered) + self.thread_pool.start(worker) + pass + + def on_home_status_gathered(self, data): + print("Works Done!") + for pfic in data: + db.update_home_status(pfic, True) pass def gather_evolution_info(self): diff --git a/ui/workers/gather_home_storage_status_worker.py b/ui/workers/gather_home_storage_status_worker.py new file mode 100644 index 0000000..23a2fb2 --- /dev/null +++ b/ui/workers/gather_home_storage_status_worker.py @@ -0,0 +1,148 @@ +from PyQt6.QtCore import QObject, pyqtSignal, QRunnable +from bs4 import BeautifulSoup +from cache import cache + +from utility.data import regions, default_forms +from utility.functions import get_objects_by_number, compare_pokemon_forms +from db import db + +class GatherHomeStorageStatusWorkerSignals(QObject): + finished = pyqtSignal(list) + +class GatherHomeStorageStatus(QRunnable): + def __init__(self): + super().__init__() + self.signals = GatherHomeStorageStatusWorkerSignals() + self.base_url = "https://www.serebii.net/pokemonhome/" + + def run(self): + try: + gathered_data = self.gather_home_storage_data() + self.signals.finished.emit(gathered_data) + except Exception as e: + print(f"Error gathering Pokémon home storage status: {e}") + + def gather_home_storage_data(self): + all_pokemon_forms = db.get_list_of_pokemon_forms() + pokemon_storable_in_home = [] + pfics_that_can_go_to_home = [] + for region in regions: + pokemon_storable_in_home.extend(self.scrape_region_for_pokemon(region)) + + for pokemon_form in all_pokemon_forms: + storable_in_home = False + name = pokemon_form["name"] + national_dex = pokemon_form["national_dex"] + working_form = pokemon_form["form_name"] + + if working_form and name in working_form: + working_form = working_form.replace(name, "").strip() + + # serebii doesn't list gender in the table so we have to assume based on form name. + if working_form and ("male" in working_form.lower() or "working_form" in working_form.lower()): + working_form = None + + if name == "Unown" and (working_form != "!" and working_form != "?"): + working_form = None + + if name == "Tauros" and working_form == "Combat Breed": + working_form = "Paldean Form" + + # serebii just gave up on Alcremie. It has 36 uniquie forms all storable in home. + if name == "Alcremie": + working_form = None + + pokemon_by_national_dex = get_objects_by_number(pokemon_storable_in_home, f"{national_dex:04d}") + for pokemon in pokemon_by_national_dex: + if working_form: + parts = pokemon['name'].split(" ") + if len(parts) > 1 and parts[0] == working_form: + storable_in_home = True + + brackets = self.extract_bracketed_text(pokemon['name']) + if brackets: + for bracket in brackets: + if name in bracket: + bracket = bracket.replace(name, "").strip() + if compare_pokemon_forms(working_form, bracket): + storable_in_home = True + break + if storable_in_home == False and working_form and working_form in default_forms: + working_form = None + + if working_form == None and name.lower() in pokemon['name'].lower(): + storable_in_home = True + break + if storable_in_home: + pfics_that_can_go_to_home.append(pokemon_form["pfic"]) + return pfics_that_can_go_to_home + + + def scrape_region_for_pokemon(self, region): + cached_entry = cache.get(f"home_{region}") + if cached_entry != None: + return cached_entry + + url = f"{self.base_url}{region}pokemon.shtml" + response = cache.fetch_url(url) + if not response: + return [] + + soup = BeautifulSoup(response, 'html.parser') + table = soup.find('table', class_='dextable') + if table == None: + return [] + + pokemon_list = [] + + rows = table.find_all('tr')[2:] # Skip the header row and the game intro row + for row in rows: + cells = row.find_all('td') + if len(cells) <= 5: # Ensure we have enough cells to check depositability. if only 5 then its not depositable in any game. + continue + + number = cells[0].text.strip().lstrip('#') + name = cells[2].text.strip() + + # Get the image URL + img_url = cells[1].find('img')['src'] + full_img_url = f"https://www.serebii.net{img_url}" + + pokemon_list.append({ + 'number': number, + 'name': name, + 'image_url': full_img_url + }) + + cache.set(f"home_{region}", pokemon_list) + + return pokemon_list + + def extract_bracketed_text(self, string): + results = [] + stack = [] + start_index = -1 + + for i, char in enumerate(string): + if char == '(': + if not stack: + start_index = i + stack.append(i) + elif char == ')': + if stack: + stack.pop() + if not stack: + results.append(string[start_index + 1:i]) + start_index = -1 + else: + #logger.warning(f"Warning: Unmatched closing parenthesis at position {i}") + pass + + # Handle any remaining unclosed brackets + if stack: + #logger.warning(f"Warning: {len(stack)} unmatched opening parentheses") + for unmatched_start in stack: + results.append(string[unmatched_start + 1:]) + + return results + diff --git a/ui/workers.py b/ui/workers/gather_pokemon_forms_worker.py similarity index 100% rename from ui/workers.py rename to ui/workers/gather_pokemon_forms_worker.py diff --git a/utility/data.py b/utility/data.py index 3973a11..b53f354 100644 --- a/utility/data.py +++ b/utility/data.py @@ -10,6 +10,7 @@ pokemon_generations = { 9: {"min": 906, "max": 1025}, } +regions = ["kanto", "johto", "hoenn", "sinnoh", "unova", "kalos", "alola", "galar", "paldea", "hisui", "unknown"] regional_descriptors = ["kantonian", "johtonian", "hoennian", "sinnohan", "unovan", "kalosian", "alolan", "galarian", "hisuian", "paldean"] yellow = { @@ -248,3 +249,57 @@ main_line_games = [ scarlet, violet, ] +# If a pokemon is in this form then its generally* not refered to as a form +# *I say generally as some do and some don't +default_forms = [ + "Male", + "Normal Forme", + "Hero of Many Battles", + "Altered Forme", + "Land Forme", + "Standard Mode", + "Ordinary Forme", + "Aria Forme", + "Natural Form", + "Shield Forme", + "Neutral Mode", + "Hoopa Confined", + "Solo Form", + "Type: Normal", + "Red Core", + "Disguised Form", + "Ice Face", + "Full Belly Mode", + "Zero Form", + "Curly Form", + "Apex Build", + "Ultimate Mode", + "Teal Mask", + "Normal Form", + "Plant Cloak", + "Overcast Form", + "West Sea", + "Normal", + "Red-Striped Form", + "Spring Form", + "Incarnate Forme", + "Meadow Pattern", + "Red Flower", + "Average Size", + "50% Forme", + "Confined", + "Baile Style", + "Midday Form", + "Amped Form", + "Vanilla Cream Strawberry Sweet", + "Single Strike Style", + "Green Plumage", + "Two-Segment Form", + "Standard Form", + "Counterfeit Form", + "Unremarkable Form", + "Antique Form", + "Phony Form", + "Masterpiece Form", + "Chest Form" +] \ No newline at end of file diff --git a/utility/functions.py b/utility/functions.py index d917bb4..840551e 100644 --- a/utility/functions.py +++ b/utility/functions.py @@ -1,5 +1,6 @@ from .data import pokemon_generations, main_line_games import unicodedata +import re def format_pokemon_id(national_dex: int, region_code: int, form_index: int, gender_code: int) -> str: return f"{national_dex:04d}-{region_code:02d}-{form_index:03d}-{gender_code}" @@ -49,4 +50,29 @@ def find_game_generation(game_name: str) -> int: for game in main_line_games: if game_name == game["Name"].lower() or game_name in (name.lower() for name in game["AltNames"]): return game["Generation"] - return None \ No newline at end of file + return None + +def sanitize_filename(filename): + # Define a dictionary of symbol replacements + symbol_replacements = { + '?': 'questionmark', + '*': 'asterisk', + ':': 'colon', + '/': 'slash', + '\\': 'backslash', + '|': 'pipe', + '<': 'lessthan', + '>': 'greaterthan', + '"': 'quote', + ' ': '_' + } + + # Replace symbols with their word equivalents + for symbol, word in symbol_replacements.items(): + filename = filename.replace(symbol, word) + + # Remove any remaining invalid characters + return re.sub(r'[<>:"/\\|?*]', '', filename) + +def get_objects_by_number(array, target_number): + return [obj for obj in array if obj['number'] == target_number] \ No newline at end of file -- 2.30.2 From 0f171bc7fe8dec0c360bd7977ab17c208558ace5 Mon Sep 17 00:00:00 2001 From: Dan Date: Tue, 5 Nov 2024 08:35:52 +0000 Subject: [PATCH 04/21] - Optimizations to the home_status gathering --- .../gather_home_storage_status_worker.py | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/ui/workers/gather_home_storage_status_worker.py b/ui/workers/gather_home_storage_status_worker.py index 23a2fb2..85351d0 100644 --- a/ui/workers/gather_home_storage_status_worker.py +++ b/ui/workers/gather_home_storage_status_worker.py @@ -24,10 +24,18 @@ class GatherHomeStorageStatus(QRunnable): def gather_home_storage_data(self): all_pokemon_forms = db.get_list_of_pokemon_forms() - pokemon_storable_in_home = [] pfics_that_can_go_to_home = [] + pokemon_by_national_dex = {} + for region in regions: - pokemon_storable_in_home.extend(self.scrape_region_for_pokemon(region)) + pokemon_list = self.scrape_region_for_pokemon(region) + for pokemon in pokemon_list: + national_dex = int(pokemon['number']) + if national_dex not in pokemon_by_national_dex: + pokemon_by_national_dex[national_dex] = [] + pokemon_by_national_dex[national_dex].append(pokemon) + + default_forms_set = set(default_forms) for pokemon_form in all_pokemon_forms: storable_in_home = False @@ -35,6 +43,9 @@ class GatherHomeStorageStatus(QRunnable): national_dex = pokemon_form["national_dex"] working_form = pokemon_form["form_name"] + if national_dex not in pokemon_by_national_dex: + continue + if working_form and name in working_form: working_form = working_form.replace(name, "").strip() @@ -42,7 +53,7 @@ class GatherHomeStorageStatus(QRunnable): if working_form and ("male" in working_form.lower() or "working_form" in working_form.lower()): working_form = None - if name == "Unown" and (working_form != "!" and working_form != "?"): + if name == "Unown" and (working_form not in ["!", "?"]): working_form = None if name == "Tauros" and working_form == "Combat Breed": @@ -52,35 +63,38 @@ class GatherHomeStorageStatus(QRunnable): if name == "Alcremie": working_form = None - pokemon_by_national_dex = get_objects_by_number(pokemon_storable_in_home, f"{national_dex:04d}") - for pokemon in pokemon_by_national_dex: + for pokemon in pokemon_by_national_dex[national_dex]: if working_form: parts = pokemon['name'].split(" ") if len(parts) > 1 and parts[0] == working_form: storable_in_home = True + break brackets = self.extract_bracketed_text(pokemon['name']) if brackets: for bracket in brackets: - if name in bracket: + if name in bracket: bracket = bracket.replace(name, "").strip() if compare_pokemon_forms(working_form, bracket): storable_in_home = True break - if storable_in_home == False and working_form and working_form in default_forms: + + if not storable_in_home and working_form in default_forms_set: working_form = None if working_form == None and name.lower() in pokemon['name'].lower(): storable_in_home = True break + if storable_in_home: pfics_that_can_go_to_home.append(pokemon_form["pfic"]) + return pfics_that_can_go_to_home def scrape_region_for_pokemon(self, region): cached_entry = cache.get(f"home_{region}") - if cached_entry != None: + if cached_entry is not None: return cached_entry url = f"{self.base_url}{region}pokemon.shtml" @@ -90,7 +104,7 @@ class GatherHomeStorageStatus(QRunnable): soup = BeautifulSoup(response, 'html.parser') table = soup.find('table', class_='dextable') - if table == None: + if table is None: return [] pokemon_list = [] -- 2.30.2 From db5661f4b3c24b274d35c6c7c9210e7debc170d7 Mon Sep 17 00:00:00 2001 From: Dan Date: Tue, 5 Nov 2024 12:56:41 +0000 Subject: [PATCH 05/21] - Made the switch to store all genders in the db if they have a unique sprite or not to help preserve the evolution lines --- database/db_controller.py | 62 ++++++++++++------ ui/main_window_controller.py | 17 +++-- ui/main_window_view.py | 5 +- ui/workers/gather_evolutions_worker.py | 29 ++++++++ ui/workers/gather_pokemon_forms_worker.py | 80 ++++++++++++++++++++--- utility/functions.py | 12 +++- 6 files changed, 167 insertions(+), 38 deletions(-) create mode 100644 ui/workers/gather_evolutions_worker.py diff --git a/database/db_controller.py b/database/db_controller.py index 2872c73..10b2fff 100644 --- a/database/db_controller.py +++ b/database/db_controller.py @@ -52,7 +52,7 @@ class DBController: ) ''') - def add_pokemon_form(self, pfic, name, form_name, national_dex, generation, sprite_url): + def add_pokemon_form(self, pfic, name, form_name, national_dex, generation, sprite_url, gender_relevant): data = { "name": name, "form_name": form_name, @@ -61,6 +61,7 @@ class DBController: "sprite_url": sprite_url, "is_baby_form": False, "storable_in_home": False, + "gender_relevant": gender_relevant } with self.lock: @@ -70,31 +71,50 @@ class DBController: self.conn.commit() print(f"Added: {pfic}, {name}") + def craft_pokemon_json_query(self, fields_to_include, pfic = None): + query = f"SELECT " + extracts = [] + for field in fields_to_include: + if field == "pfic": + extracts.append("PFIC as pfic") + else: + extracts.append(f"JSON_EXTRACT(data, '$.{field}') AS {field}") + query = query + ", ".join(extracts) + query = query + " FROM pokemon_forms" + + if pfic is not None: + query = query + f" WHERE PFIC = '{pfic}'" + + return query + def get_pokemon_details(self, pfic): - self.cursor.execute(''' - SELECT JSON_EXTRACT(data, '$.name') AS name, - JSON_EXTRACT(data, '$.form_name') AS form_name, - JSON_EXTRACT(data, '$.national_dex') AS national_dex, - JSON_EXTRACT(data, '$.generation') AS generation, - JSON_EXTRACT(data, '$.is_baby_form') AS is_baby_form, - JSON_EXTRACT(data, '$.storable_in_home') AS storable_in_home - FROM pokemon_forms - WHERE PFIC = ? - ''', (pfic,)) + fields = [ + "name", + "form_name", + "national_dex", + "generation", + "is_baby_form", + "storable_in_home", + ] + query = self.craft_pokemon_json_query(fields, pfic) + self.cursor.execute(query) results = self.cursor.fetchone() return dict(results) def get_list_of_pokemon_forms(self): - self.cursor.execute(''' - SELECT JSON_EXTRACT(data, '$.name') AS name, - JSON_EXTRACT(data, '$.form_name') AS form_name, - JSON_EXTRACT(data, '$.national_dex') AS national_dex, - JSON_EXTRACT(data, '$.generation') AS generation, - JSON_EXTRACT(data, '$.is_baby_form') AS is_baby_form, - JSON_EXTRACT(data, '$.storable_in_home') AS storable_in_home, - PFIC as pfic - FROM pokemon_forms - ''',) + fields = [ + "pfic", + "name", + "form_name", + "national_dex", + "generation", + "is_baby_form", + "storable_in_home", + "gender_relevant" + ] + + query = self.craft_pokemon_json_query(fields) + self.cursor.execute(query) results = self.cursor.fetchall() return [dict(row) for row in results] diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index 7e20b75..bb09c90 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -6,6 +6,7 @@ import os from ui.workers.gather_home_storage_status_worker import GatherHomeStorageStatus from ui.workers.gather_pokemon_forms_worker import GatherPokemonFormsWorker +from utility.functions import get_display_name from db import db class MainWindowController: @@ -21,6 +22,7 @@ class MainWindowController: def initialize_pokemon_list(self, data): self.pokemon_data_cache = data self.view.update_pokemon_forms(data) + self.apply_filters() def filter_pokemon_list(self): self.filter_timer.start() @@ -29,9 +31,12 @@ class MainWindowController: search_text = self.view.search_bar.text().lower() show_only_home_storable = self.view.filter_home_storable.isChecked() show_only_missing_encounters = self.view.highlight_no_encounters.isChecked() + gender_relevant = False filtered_data = [] - for pfic, display_name in self.pokemon_data_cache: + for pokemon in self.pokemon_data_cache: + display_name = get_display_name(pokemon) + pfic = pokemon["pfic"] # Check if the item matches the search text text_match = search_text in display_name.lower() @@ -47,9 +52,13 @@ class MainWindowController: # TODO: reimplement this check. has_encounters = True + include_gender = True + if gender_relevant == False and pokemon["gender_relevant"] == False: + include_gender = not any(item["pfic"][:-2] == pfic[:-2] for item in filtered_data) + # If both conditions are met, add to filtered data - if text_match and home_storable: - filtered_data.append((pfic, display_name)) + if text_match and home_storable and include_gender: + filtered_data.append(pokemon) # Update the view with the filtered data self.view.update_pokemon_forms(filtered_data) @@ -91,7 +100,7 @@ class MainWindowController: # This method will be called in the main thread when the worker finishes # Update the UI with the gathered forms for pokemon in data: - db.add_pokemon_form(pokemon["pfic"], pokemon["name"], pokemon["form_name"], pokemon["national_dex"], pokemon["generation"], pokemon["sprite_url"]) + db.add_pokemon_form(pokemon["pfic"], pokemon["name"], pokemon["form_name"], pokemon["national_dex"], pokemon["generation"], pokemon["sprite_url"], pokemon["gender_relevant"]) self.pokemon_data_cache = data self.view.update_pokemon_forms(data) diff --git a/ui/main_window_view.py b/ui/main_window_view.py index 5862b6f..12601f4 100644 --- a/ui/main_window_view.py +++ b/ui/main_window_view.py @@ -6,6 +6,7 @@ from PyQt6.QtCore import Qt, QSize, QTimer, QMetaObject from PyQt6.QtGui import QPixmap, QFontMetrics, QColor, QAction from .main_window_controller import MainWindowController +from utility.functions import get_display_name from db import db class PokemonUI(QWidget): @@ -217,9 +218,7 @@ class PokemonUI(QWidget): self.pokemon_list.clear() for pokemon in data: - display_name = f"{pokemon["national_dex"]:04d} - {pokemon["name"]}" - if pokemon["form_name"]: - display_name += f" ({pokemon["form_name"]})" + display_name = get_display_name(pokemon, not pokemon["gender_relevant"]) item = QListWidgetItem(display_name) item.setData(Qt.ItemDataRole.UserRole, pokemon["pfic"]) self.pokemon_list.addItem(item) \ No newline at end of file diff --git a/ui/workers/gather_evolutions_worker.py b/ui/workers/gather_evolutions_worker.py new file mode 100644 index 0000000..6953380 --- /dev/null +++ b/ui/workers/gather_evolutions_worker.py @@ -0,0 +1,29 @@ +from PyQt6.QtCore import QObject, pyqtSignal, QRunnable +from bs4 import BeautifulSoup +from cache import cache +from db import db + +class GatherEvolutionsWorkerSignals(QObject): + finished = pyqtSignal(list) + +class GatherHEvolutions(QRunnable): + def __init__(self): + super().__init__() + self.signals = GatherEvolutionsWorkerSignals() + self.base_url = "https://www.serebii.net/pokemonhome/" + + def run(self): + try: + gathered_data = self.gather_evolution_data() + self.signals.finished.emit(gathered_data) + except Exception as e: + print(f"Error gathering Pokémon home storage status: {e}") + + def gather_evolution_data(self): + all_pokemon_forms = db.get_list_of_pokemon_forms() + evolutions = [] + + for pokemon_form in all_pokemon_forms: + pass + + return evolutions \ No newline at end of file diff --git a/ui/workers/gather_pokemon_forms_worker.py b/ui/workers/gather_pokemon_forms_worker.py index 470753e..0a602d0 100644 --- a/ui/workers/gather_pokemon_forms_worker.py +++ b/ui/workers/gather_pokemon_forms_worker.py @@ -108,15 +108,21 @@ class GatherPokemonFormsWorker(QRunnable): form_name = self.extract_form_name(sprite) #logger.info(f'{sprite_url}, {form_name}') + record_male_form = False + record_female_form = False + record_genderless_form = False + gender_relevant = False if form_name != "None": form_index += 1 gender = 0 if form_name.startswith("Male"): form_index -= 1 gender = 1 + gender_relevant = True elif form_name.startswith("Female"): form_index -= 1 gender = 2 + gender_relevant = True dex_page_data = self.get_pokemon_dex_page(url_name) if dex_page_data: @@ -156,15 +162,71 @@ class GatherPokemonFormsWorker(QRunnable): if generation_found: break - pokemon_form = { - "pfic":format_pokemon_id(national_dex_number, generation, form_index, gender), - "name":pokemon_name, - "form_name":form_name if form_name != "None" else None, - "sprite_url":sprite_url, - "national_dex":national_dex_number, - "generation":generation - } - found_forms.append(pokemon_form) + if not gender_relevant: + # see if we can find gender info on the page to see if it has male and female forms anyway. + gender_header = dex_soup.find('th', string="Gender") + if gender_header: + gender_info = gender_header.findNext('td').getText().replace(",", "").split() + skip_next = False + for info in gender_info: + if skip_next: + skip_next = False + continue + if info.lower().startswith("0%"): + skip_next = True + continue + if info.lower() == "male": + record_male_form = True + elif info.lower() == "female": + record_female_form = True + + if not record_female_form and not record_male_form: + record_genderless_form = True + + if gender_relevant or record_genderless_form: + pokemon_form = { + "pfic":format_pokemon_id(national_dex_number, generation, form_index, gender), + "name":pokemon_name, + "form_name":form_name if form_name != "None" else None, + "sprite_url":sprite_url, + "national_dex":national_dex_number, + "generation":generation, + "gender_relevant": gender_relevant + } + found_forms.append(pokemon_form) + else: + if record_male_form: + gendered_form = form_name + if gendered_form == "None": + gendered_form = "Male" + else: + gendered_form = "Male " + gendered_form + pokemon_form = { + "pfic":format_pokemon_id(national_dex_number, generation, form_index, 1), + "name":pokemon_name, + "form_name":gendered_form if gendered_form != "None" else None, + "sprite_url":sprite_url, + "national_dex":national_dex_number, + "generation":generation, + "gender_relevant": gender_relevant + } + found_forms.append(pokemon_form) + if record_female_form: + gendered_form = form_name + if gendered_form == "None": + gendered_form = "Female" + else: + gendered_form = "Female " + gendered_form + pokemon_form = { + "pfic":format_pokemon_id(national_dex_number, generation, form_index, 2), + "name":pokemon_name, + "form_name":gendered_form if gendered_form != "None" else None, + "sprite_url":sprite_url, + "national_dex":national_dex_number, + "generation":generation, + "gender_relevant": gender_relevant + } + found_forms.append(pokemon_form) cache.set(url_name, found_forms) return found_forms \ No newline at end of file diff --git a/utility/functions.py b/utility/functions.py index 840551e..4d6dfe2 100644 --- a/utility/functions.py +++ b/utility/functions.py @@ -75,4 +75,14 @@ def sanitize_filename(filename): return re.sub(r'[<>:"/\\|?*]', '', filename) def get_objects_by_number(array, target_number): - return [obj for obj in array if obj['number'] == target_number] \ No newline at end of file + return [obj for obj in array if obj['number'] == target_number] + +def get_display_name(pokemon, strip_gender = False): + display_name = f"{pokemon["national_dex"]:04d} - {pokemon["name"]}" + if pokemon["form_name"]: + form = pokemon["form_name"] + if strip_gender: + form = form.replace("Female", "").replace("Male", "").strip() + if form != "": + display_name += f" ({form})" + return display_name \ No newline at end of file -- 2.30.2 From f14bee7279a442de4761a86ef27482d3527ae009 Mon Sep 17 00:00:00 2001 From: Dan Date: Tue, 5 Nov 2024 15:06:04 +0000 Subject: [PATCH 06/21] - Start work on the new evolution management and processing --- ui/main_window_controller.py | 11 +- ui/workers/gather_evolutions_worker.py | 217 ++++++++++++++++++++++++- utility/functions.py | 10 +- 3 files changed, 228 insertions(+), 10 deletions(-) diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index bb09c90..e20ce15 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -5,6 +5,7 @@ import os from ui.workers.gather_home_storage_status_worker import GatherHomeStorageStatus from ui.workers.gather_pokemon_forms_worker import GatherPokemonFormsWorker +from ui.workers.gather_evolutions_worker import GatherEvolutions from utility.functions import get_display_name from db import db @@ -110,16 +111,20 @@ class MainWindowController: worker = GatherHomeStorageStatus() worker.signals.finished.connect(self.on_home_status_gathered) self.thread_pool.start(worker) - pass def on_home_status_gathered(self, data): print("Works Done!") for pfic in data: db.update_home_status(pfic, True) - pass + def gather_evolution_info(self): - pass + worker = GatherEvolutions() + worker.signals.finished.connect(self.on_evolutions_gathered) + self.thread_pool.start(worker) + + def on_evolutions_gathered(self, data): + print("Works Done!") def reinitialize_database(self): pass diff --git a/ui/workers/gather_evolutions_worker.py b/ui/workers/gather_evolutions_worker.py index 6953380..3bc5844 100644 --- a/ui/workers/gather_evolutions_worker.py +++ b/ui/workers/gather_evolutions_worker.py @@ -1,16 +1,19 @@ +from typing import Optional from PyQt6.QtCore import QObject, pyqtSignal, QRunnable -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup, Tag from cache import cache from db import db +from utility.functions import get_form_name, get_display_name + class GatherEvolutionsWorkerSignals(QObject): finished = pyqtSignal(list) -class GatherHEvolutions(QRunnable): +class GatherEvolutions(QRunnable): def __init__(self): super().__init__() self.signals = GatherEvolutionsWorkerSignals() - self.base_url = "https://www.serebii.net/pokemonhome/" + self.base_url = "https://bulbapedia.bulbagarden.net/wiki/" def run(self): try: @@ -24,6 +27,210 @@ class GatherHEvolutions(QRunnable): evolutions = [] for pokemon_form in all_pokemon_forms: - pass + print(f"Processing {get_display_name(pokemon_form)}'s evolutions") + url = f"https://bulbapedia.bulbagarden.net/wiki/{pokemon_form["name"]}_(Pokémon)" + page_data = cache.fetch_url(url) + if not page_data: + continue + soup = BeautifulSoup(page_data, 'html.parser') + evolution_section = soup.find('span', id='Evolution_data') + if not evolution_section: + continue + evolution_table = None + form = get_form_name(pokemon_form, not pokemon_form["gender_relevant"]) + pokemon_name = pokemon_form["name"] + evolution_table = evolution_section.parent.find_next('table') + if form: + form_without_form = form.replace('Form', '').replace('form', '').strip() + for tag in evolution_section.parent.find_next_siblings(): + if tag.name == 'h4' and form_without_form in tag.get_text(strip=True): + evolution_table = tag.find_next('table') + break + if tag.name == 'h3': + break + if not evolution_table: + continue + + if pokemon_name == "Eevee": + evolution_chain = self.parse_eevee_evolution_chain(evolution_table, pokemon_form) + evolutions.append(evolution_chain) + else: + evolution_chain = self.parse_evolution_chain(evolution_table, pokemon_form) + evolutions.append(evolution_chain) + + return evolutions + + def parse_evolution_chain(self, table, pokemon_form, force_refresh = False): + cache_record_name = f"evo_{pokemon_form["pfic"]}" + if force_refresh: + cache.purge(cache_record_name) + + cached_entry = cache.get(cache_record_name) + if cached_entry != None: + return cached_entry + + main_chain = [] + current_stage = None + pending_method = None + form = get_form_name(pokemon_form, not pokemon_form["gender_relevant"]) + + tbody = table.find('tbody', recursive=False) + if not tbody: + return [] + + rows = tbody.find_all('tr', recursive=False) + main_row = rows[0] + branch_rows = rows[1:] + + # Parse main evolution chain + for td in main_row.find_all('td', recursive=False): + if td.find('table'): + # This TD contains Pokemon information + pokemon_name = self.extract_pokemon_name(td) + stage = self.extract_stage_form(td) + evolution_form = self.extract_evolution_form(td, pokemon_name) + new_stage = { + "pokemon":pokemon_name, + "method": pending_method, + "stage": stage, + "form": evolution_form, + "next_stage": None, + "previous_stage": None, + "branches": [], + "pfic": pokemon_form["pfic"] + } + pending_method = None + if current_stage: + current_stage["next_stage"] = new_stage + new_stage["previous_stage"] = current_stage # Set the back link + current_stage = new_stage + main_chain.append(current_stage) + else: + # This TD contains evolution method for the next Pokemon + pending_method = self.extract_evolution_method(td) + + # Parse branching evolutions + for row in branch_rows: + branch_stage = None + branch_method = None + for td in row.find_all('td', recursive=False): + if td.find('table'): + pokemon_name = self.extract_pokemon_name(td) + stage = self.extract_stage_form(td) + evolution_form = self.extract_evolution_form(td, pokemon_name) + new_stage = { + "pokemon":pokemon_name, + "method": branch_method, + "stage": stage, + "form": evolution_form, + "next_stage": None, + "previous_stage": None, + "branches": [], + "pfic": pokemon_form["pfic"] + } + branch_method = None + if branch_stage: + branch_stage["next_stage"] = new_stage + new_stage["previous_stage"] = branch_stage # Set the back link + branch_stage = new_stage + # Find which main chain Pokemon this branches from + for main_stage in main_chain: + if td.get('rowspan') and main_stage.pokemon == pokemon_name: + main_stage["branches"].append(branch_stage) + branch_stage["previous_stage"] = main_stage # Set the back link to the main chain + break + else: + branch_method = self.extract_evolution_method(td) + + cache.set(cache_record_name, main_chain) + return main_chain + + def extract_pokemon_name(self, td: Tag) -> Optional[str]: + name_tag = self.find_name_tag(td) + if name_tag: + return name_tag.get_text(strip=True) + return None + + def find_name_tag(self, td: Tag) -> Optional[Tag]: + table = td.find('table') + name_tag = table.find('a', class_='selflink') + if name_tag: + return name_tag + name_tag = table.find('a', title=True, class_=lambda x: x != 'image') + return name_tag + + def extract_stage_form(self, td: Tag) -> Optional[str]: + stage_tag = td.find('table').find('small') + if stage_tag: + return stage_tag.get_text(strip=True) + return None + + def extract_evolution_form(self, td: Tag, name: str) -> Optional[str]: + name_tag = self.find_name_tag(td) + if name_tag: + name_row = name_tag.parent + small_tags = name_row.find_all('small') + if len(small_tags) > 1: + return small_tags[0].get_text(strip=True) + return None + + def extract_evolution_method(self, td: Tag) -> str: + # Extract evolution method from the TD + return td.get_text(strip=True) + + def parse_eevee_evolution_chain(self, table, pokemon_form): + tbody = table.find('tbody', recursive=False) + if not tbody: + return [] + + rows = tbody.find_all('tr', recursive=False) + eevee_row = rows[1] + method_row = rows[2] + eeveelutions_row = rows[3] + + eevee_td = eevee_row.find('td', recursive=False) + pokemon_name, stage = self.parse_pokemon_subtable(eevee_td) + eevee_stage = { + "pokemon":pokemon_name, + "method": None, + "stage": stage, + "form": None, + "next_stage": None, + "previous_stage": None, + "branches": [], + "pfic": pokemon_form["pfic"] + } + + methods = [] + for method in method_row.find_all('td', recursive=False): + methods.append(self.extract_evolution_method(method)) + + eeveelutions = [] + index = 0 + for eeveelution in eeveelutions_row.find_all('td', recursive=False): + pokemon_name, stage = self.parse_pokemon_subtable(eeveelution) + eeveelution_stage = { + "pokemon":pokemon_name, + "method": methods[index], + "stage": stage, + "form": None, + "next_stage": None, + "previous_stage": None, + "branches": [], + "pfic": pokemon_form["pfic"] + } + eeveelution_stage["previous_stage"] = eevee_stage # Set the back link to Eevee + eeveelutions.append(eeveelution_stage) + index += 1 + + eevee_stage["branches"] = eeveelutions # Set the branches directly, not as a nested list - return evolutions \ No newline at end of file + return [eevee_stage] + + def parse_pokemon_subtable(self, td): + if td.find('table'): + # This TD contains Pokemon information + pokemon_name = self.extract_pokemon_name(td) + stage = self.extract_stage_form(td) + return pokemon_name, stage + return None, None \ No newline at end of file diff --git a/utility/functions.py b/utility/functions.py index 4d6dfe2..b54d010 100644 --- a/utility/functions.py +++ b/utility/functions.py @@ -79,10 +79,16 @@ def get_objects_by_number(array, target_number): def get_display_name(pokemon, strip_gender = False): display_name = f"{pokemon["national_dex"]:04d} - {pokemon["name"]}" + form = get_form_name(pokemon, strip_gender) + if form: + display_name += f" ({form})" + return display_name + +def get_form_name(pokemon, strip_gender = False): if pokemon["form_name"]: form = pokemon["form_name"] if strip_gender: form = form.replace("Female", "").replace("Male", "").strip() if form != "": - display_name += f" ({form})" - return display_name \ No newline at end of file + return form + return None \ No newline at end of file -- 2.30.2 From 706b6303205693afdae84fd07c28a6a2d90e5966 Mon Sep 17 00:00:00 2001 From: Quildra Date: Tue, 5 Nov 2024 21:35:43 +0000 Subject: [PATCH 07/21] - WIP new node graph tech for evolutions. - Need to refine the chains, there are duplicates --- database/db_controller.py | 73 +++++++++++ ui/main_window_controller.py | 7 +- ui/main_window_view.py | 41 ++++++- ui/workers/gather_evolutions_worker.py | 142 ++++++++++++++++++++-- ui/workers/gather_pokemon_forms_worker.py | 2 +- 5 files changed, 253 insertions(+), 12 deletions(-) diff --git a/database/db_controller.py b/database/db_controller.py index 10b2fff..0eca676 100644 --- a/database/db_controller.py +++ b/database/db_controller.py @@ -1,6 +1,8 @@ import sqlite3 import threading import json +import os +import networkx as nx class DBController: def __init__(self, db_path=':memory:', max_connections=10): @@ -9,6 +11,7 @@ class DBController: self.conn = sqlite3.connect(db_path, check_same_thread=False) self.conn.row_factory = sqlite3.Row self.cursor = self.conn.cursor() + self.graph = nx.DiGraph() self.init_database() def init_database(self): @@ -27,6 +30,11 @@ class DBController: # Close the file-based database connection disk_conn.close() + if os.path.exists("pokemon_evolution_graph.json"): + with open("pokemon_evolution_graph.json", "r") as f: + data = json.load(f) + self.graph = nx.node_link_graph(data) + def save_changes(self): with self.lock: # Count the number of records before backup for verification @@ -40,6 +48,10 @@ class DBController: self.conn.backup(disk_conn) disk_conn.close() + data = nx.node_link_data(self.graph) + with open("pokemon_evolution_graph.json", "w") as f: + json.dump(data, f) + def close(self): self.save_changes() self.conn.close() @@ -95,12 +107,21 @@ class DBController: "generation", "is_baby_form", "storable_in_home", + "gender_relevant" ] query = self.craft_pokemon_json_query(fields, pfic) self.cursor.execute(query) results = self.cursor.fetchone() return dict(results) + def get_pokemon_details_by_name(self, name, fields): + query = self.craft_pokemon_json_query(fields) + name = name.replace("'", "''") + query += f" WHERE JSON_EXTRACT(data, '$.name') = '{name}'" + self.cursor.execute(query) + results = self.cursor.fetchall() + return [dict(row) for row in results] + def get_list_of_pokemon_forms(self): fields = [ "pfic", @@ -141,3 +162,55 @@ class DBController: WHERE PFIC = ? ''', (updated_data_str, pfic)) self.conn.commit() + + def update_evolution_graph(self, evolutions): + for evolution in evolutions: + from_pfic = evolution["from_pfic"] + to_pfic = evolution["to_pfic"] + method = evolution["method"] + + # Add nodes if they do not already exist + if not self.graph.has_node(from_pfic): + self.graph.add_node(from_pfic) + + if not self.graph.has_node(to_pfic): + self.graph.add_node(to_pfic) + + # Add the edge representing the evolution, with the method as an attribute + self.graph.add_edge(from_pfic, to_pfic, method=method) + + def get_evolution_graph(self, pfic): + return list(self.graph.successors(pfic)) + + def get_evolution_paths(self, start_node): + paths = [] + + # Define a recursive function to traverse the graph + def traverse(current_node, current_path): + # Add the current node to the path + current_path.append(current_node) + + # Get successors of the current node + successors = list(self.graph.successors(current_node)) + + if not successors: + # If there are no successors, add the current path to paths list + paths.append(current_path.copy()) + else: + # Traverse each successor and add edge metadata + for successor in successors: + method = self.graph[current_node][successor]["method"] + # Add the edge metadata as a tuple (to_node, method) + current_path.append((successor, method)) + + # Recur for the successor + traverse(successor, current_path) + + # Backtrack (remove the last node and edge metadata) + current_path.pop() + current_path.pop() + + # Start traversal from the start_node + traverse(start_node, []) + + return paths diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index e20ce15..8723d36 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -125,6 +125,7 @@ class MainWindowController: def on_evolutions_gathered(self, data): print("Works Done!") + db.update_evolution_graph(data) def reinitialize_database(self): pass @@ -168,8 +169,10 @@ class MainWindowController: else: self.view.image_label.setText("Image not found") - #self.load_evolution_chain(pfic) + self.load_evolution_chain(pfic) #self.load_encounter_locations(pfic) self.current_pfic = pfic - + def load_evolution_chain(self, pfic): + chain = db.get_evolution_paths(pfic) + self.view.update_evolution_tree(chain, pfic) diff --git a/ui/main_window_view.py b/ui/main_window_view.py index 12601f4..fafdc6c 100644 --- a/ui/main_window_view.py +++ b/ui/main_window_view.py @@ -221,4 +221,43 @@ class PokemonUI(QWidget): display_name = get_display_name(pokemon, not pokemon["gender_relevant"]) item = QListWidgetItem(display_name) item.setData(Qt.ItemDataRole.UserRole, pokemon["pfic"]) - self.pokemon_list.addItem(item) \ No newline at end of file + self.pokemon_list.addItem(item) + + def update_evolution_tree(self, evolution_chain, selected_pfic): + tree_items = {} + #for item in evolution_chain: + # print(item) + for pfic in evolution_chain: + pokemon_details = db.get_pokemon_details(pfic) + display_name = get_display_name(pokemon_details, not pokemon_details["gender_relevant"]) + item = QTreeWidgetItem([display_name, method if method else ""]) + item.setData(0, Qt.ItemDataRole.UserRole, current_pfic) + tree_items[current_pfic] = item + + if current_pfic == selected_pfic: + item.setBackground(0, QColor(255, 255, 0, 100)) # Highlight selected Pokémon + + # Second pass: build the tree structure + root = None + for current_pfic, name, form_name, method in evolution_chain: + item = tree_items[current_pfic] + + # Find the parent of this item + #parent_pfic = event_system.call_sync('get_evolution_parent', data=current_pfic) + parent_pfic = None + + if parent_pfic: + parent_item = tree_items.get(parent_pfic[0]) + if parent_item: + parent_item.addChild(item) + elif not root: + root = item + self.evolution_tree.addTopLevelItem(root) + + # Expand the entire tree + self.evolution_tree.expandAll() + + # Scroll to and select the current Pokémon + current_item = tree_items[selected_pfic] + self.evolution_tree.scrollToItem(current_item) + self.evolution_tree.setCurrentItem(current_item) \ No newline at end of file diff --git a/ui/workers/gather_evolutions_worker.py b/ui/workers/gather_evolutions_worker.py index 3bc5844..fad179e 100644 --- a/ui/workers/gather_evolutions_worker.py +++ b/ui/workers/gather_evolutions_worker.py @@ -1,10 +1,13 @@ from typing import Optional from PyQt6.QtCore import QObject, pyqtSignal, QRunnable from bs4 import BeautifulSoup, Tag +from fuzzywuzzy import fuzz from cache import cache from db import db -from utility.functions import get_form_name, get_display_name +import re + +from utility.functions import get_form_name, get_display_name, parse_pfic class GatherEvolutionsWorkerSignals(QObject): finished = pyqtSignal(list) @@ -22,23 +25,50 @@ class GatherEvolutions(QRunnable): except Exception as e: print(f"Error gathering Pokémon home storage status: {e}") - def gather_evolution_data(self): + def gather_evolution_data(self, force_refresh = False): all_pokemon_forms = db.get_list_of_pokemon_forms() evolutions = [] for pokemon_form in all_pokemon_forms: print(f"Processing {get_display_name(pokemon_form)}'s evolutions") - url = f"https://bulbapedia.bulbagarden.net/wiki/{pokemon_form["name"]}_(Pokémon)" + pokemon_name = pokemon_form["name"] + form = get_form_name(pokemon_form) + + cache_record_name = f"chain_{pokemon_name}_{form}" + if force_refresh: + cache.purge(cache_record_name) + + cached_entry = cache.get(cache_record_name) + if cached_entry != None: + evolutions.extend(cached_entry) + continue + + #form = get_form_name(pokemon_form, not pokemon_form["gender_relevant"]) + search_form = form + if search_form and pokemon_name in search_form: + search_form = search_form.replace(pokemon_name, "").strip() + + gender = None + if search_form and "male" in search_form.lower(): + gender = search_form + search_form = None + + if pokemon_name == "Flabébé": + # Bulbapedia doesn't detail out Flabébé's evolution chain fully. as its exactly the same for each form, but the coloured form remains constant + # through the evolution line, Red->Red->Red, Yellow->Yellow->Yellow etc. + search_form = None + + url = f"https://bulbapedia.bulbagarden.net/wiki/{pokemon_name}_(Pokémon)" page_data = cache.fetch_url(url) if not page_data: continue + soup = BeautifulSoup(page_data, 'html.parser') evolution_section = soup.find('span', id='Evolution_data') if not evolution_section: continue + evolution_table = None - form = get_form_name(pokemon_form, not pokemon_form["gender_relevant"]) - pokemon_name = pokemon_form["name"] evolution_table = evolution_section.parent.find_next('table') if form: form_without_form = form.replace('Form', '').replace('form', '').strip() @@ -51,12 +81,54 @@ class GatherEvolutions(QRunnable): if not evolution_table: continue + evolution_chain = [] if pokemon_name == "Eevee": evolution_chain = self.parse_eevee_evolution_chain(evolution_table, pokemon_form) - evolutions.append(evolution_chain) + #evolutions.append(evolution_chain) else: evolution_chain = self.parse_evolution_chain(evolution_table, pokemon_form) - evolutions.append(evolution_chain) + #evolutions.append(evolution_chain) + + chain = [] + for pokemon in evolution_chain: + from_pfic = self.get_pokemon_form_by_name(pokemon["pokemon"], pokemon["form"], gender=gender) + if not from_pfic: + #logger.warning(f"Could not find PFIC for {stage.pokemon} {stage.form}") + continue + + stage = pokemon["next_stage"] + if stage: + to_pfic = self.get_pokemon_form_by_name(stage["pokemon"], stage["form"], gender=gender) + if to_pfic: + evolution_info = { + "from_pfic": from_pfic, + "to_pfic": to_pfic, + "method": stage["method"] + } + evolutions.append(evolution_info) + chain.append(evolution_info) + + #insert_evolution_info(evolution_info) + + #if "breed" in stage["next_stage"]["method"].lower(): + # update_pokemon_baby_status(from_pfic, True) + + for branch in pokemon["branches"]: + to_pfic = self.get_pokemon_form_by_name(branch["pokemon"], branch["form"], gender=gender) + if to_pfic: + evolution_info = { + "from_pfic": from_pfic, + "to_pfic": to_pfic, + "method": branch["method"] + } + evolutions.append(evolution_info) + chain.append(evolution_info) + #EvolutionInfo(from_pfic, to_pfic, branch.method) + #insert_evolution_info(evolution_info) + + #if "breed" in branch.method.lower(): + # update_pokemon_baby_status(from_pfic, True) + cache.set(cache_record_name, chain) return evolutions @@ -233,4 +305,58 @@ class GatherEvolutions(QRunnable): pokemon_name = self.extract_pokemon_name(td) stage = self.extract_stage_form(td) return pokemon_name, stage - return None, None \ No newline at end of file + return None, None + + def get_pokemon_form_by_name(self, name: str, form: Optional[str] = None, threshold: int = 80, gender: Optional[str] = None): + fields = [ + "pfic", + "name", + "form_name" + ] + results = db.get_pokemon_details_by_name(name, fields) + #results = db_controller.execute_query('SELECT PFIC, name, form_name FROM pokemon_forms WHERE name = ?', (name,)) + + if not results: + return None + + results.sort(key=lambda x: parse_pfic(x["pfic"])) + + if form is None and gender is None: + if len(results) > 1: + if results[0]["form_name"] == None: + return results[0]["pfic"] + else: + return self.get_pokemon_form_by_name(name, "Male", threshold=100, gender=gender) + else: + return results[0]["pfic"] # Return the PFIC of the first result if no form is specified + + if gender: + gendered_form = self.get_pokemon_form_by_name(name, gender, threshold=100) + if gendered_form: + return gendered_form + + stripped_form = self.strip_pokemon_name(name, form) + + for entry in results: + stripped_db_form = self.strip_pokemon_name(entry["name"], entry["form_name"]) + if self.fuzzy_match_form(stripped_form, stripped_db_form, threshold): + return entry["pfic"] + + # Some times we get a form for a pokemon that doesn't really have one. + if len(results) > 1 and form != None: + return results[0]["pfic"] + + return None + + def strip_pokemon_name(self, pokemon_name: str, form_name: str) -> str: + if form_name: + form_name = form_name.replace("Form", "").strip() + form_name = re.sub(f'{re.escape(pokemon_name)}\\s*', '', form_name, flags=re.IGNORECASE).strip() + form_name = form_name.replace(" ", " ") + return form_name + return form_name + + def fuzzy_match_form(self, form1: str, form2: str, threshold: int = 80) -> bool: + if form1 is None or form2 is None: + return form1 == form2 + return fuzz.ratio(form1.lower(), form2.lower()) >= threshold \ No newline at end of file diff --git a/ui/workers/gather_pokemon_forms_worker.py b/ui/workers/gather_pokemon_forms_worker.py index 0a602d0..e41fec6 100644 --- a/ui/workers/gather_pokemon_forms_worker.py +++ b/ui/workers/gather_pokemon_forms_worker.py @@ -59,7 +59,7 @@ class GatherPokemonFormsWorker(QRunnable): return form_name return "None" - def process_pokemon_entry(self, national_dex_number, pokemon_soup, force_refresh = False): + def process_pokemon_entry(self, national_dex_number, pokemon_soup, force_refresh = True): found_forms = [] generation = get_generation_from_national_dex(national_dex_number) pokemon_name = pokemon_soup.get_text(strip=True) -- 2.30.2 From 8dd19ad2499cf8982881a2560e95d945a45a7822 Mon Sep 17 00:00:00 2001 From: Dan Date: Wed, 6 Nov 2024 11:03:15 +0000 Subject: [PATCH 08/21] - WIP re-working evolution chain parsing --- ui/workers/gather_evolutions_worker.py | 199 ++++++++++++++----------- utility/data.py | 6 + 2 files changed, 122 insertions(+), 83 deletions(-) diff --git a/ui/workers/gather_evolutions_worker.py b/ui/workers/gather_evolutions_worker.py index fad179e..712e424 100644 --- a/ui/workers/gather_evolutions_worker.py +++ b/ui/workers/gather_evolutions_worker.py @@ -8,6 +8,7 @@ from db import db import re from utility.functions import get_form_name, get_display_name, parse_pfic +from utility.data import non_evolution_forms class GatherEvolutionsWorkerSignals(QObject): finished = pyqtSignal(list) @@ -25,23 +26,27 @@ class GatherEvolutions(QRunnable): except Exception as e: print(f"Error gathering Pokémon home storage status: {e}") - def gather_evolution_data(self, force_refresh = False): + def gather_evolution_data(self, force_refresh = True): all_pokemon_forms = db.get_list_of_pokemon_forms() - evolutions = [] + #evolutions = [] + evolutions = {} for pokemon_form in all_pokemon_forms: print(f"Processing {get_display_name(pokemon_form)}'s evolutions") pokemon_name = pokemon_form["name"] form = get_form_name(pokemon_form) + if pokemon_form["form_name"] and any(s in pokemon_form["form_name"] for s in non_evolution_forms): + continue + cache_record_name = f"chain_{pokemon_name}_{form}" if force_refresh: cache.purge(cache_record_name) - cached_entry = cache.get(cache_record_name) - if cached_entry != None: - evolutions.extend(cached_entry) - continue + #cached_entry = cache.get(cache_record_name) + #if cached_entry != None: + # evolutions[pokemon_form["pfic"]] = cached_entry + # continue #form = get_form_name(pokemon_form, not pokemon_form["gender_relevant"]) search_form = form @@ -82,13 +87,17 @@ class GatherEvolutions(QRunnable): continue evolution_chain = [] + evolution_tree = None if pokemon_name == "Eevee": - evolution_chain = self.parse_eevee_evolution_chain(evolution_table, pokemon_form) + evolution_tree = self.parse_eevee_evolution_chain(evolution_table, pokemon_form) #evolutions.append(evolution_chain) else: - evolution_chain = self.parse_evolution_chain(evolution_table, pokemon_form) + evolution_tree = self.parse_evolution_chain(evolution_table, pokemon_form) #evolutions.append(evolution_chain) + if evolution_tree: + self.traverse_and_store(evolution_tree, evolutions, gender) + chain = [] for pokemon in evolution_chain: from_pfic = self.get_pokemon_form_by_name(pokemon["pokemon"], pokemon["form"], gender=gender) @@ -105,7 +114,7 @@ class GatherEvolutions(QRunnable): "to_pfic": to_pfic, "method": stage["method"] } - evolutions.append(evolution_info) + evolutions[pokemon_form["pfic"]] = evolution_info chain.append(evolution_info) #insert_evolution_info(evolution_info) @@ -121,7 +130,7 @@ class GatherEvolutions(QRunnable): "to_pfic": to_pfic, "method": branch["method"] } - evolutions.append(evolution_info) + evolutions[pokemon_form["pfic"]] = evolution_info chain.append(evolution_info) #EvolutionInfo(from_pfic, to_pfic, branch.method) #insert_evolution_info(evolution_info) @@ -132,90 +141,102 @@ class GatherEvolutions(QRunnable): return evolutions + def traverse_and_store(self, node, evolutions, gender): + """Helper function to traverse evolution tree and store evolutions.""" + from_pfic = self.get_pokemon_form_by_name(node["pokemon"], node["form"], gender=gender) + if not from_pfic: + return + + for next_stage in node["evolves_to"]: + to_pfic = self.get_pokemon_form_by_name(next_stage["pokemon"], next_stage["form"], gender=gender) + if to_pfic: + composite_key = f"{from_pfic}->{to_pfic}" + evolution_info = { + "from_pfic": from_pfic, + "to_pfic": to_pfic, + "method": next_stage["method"] + } + evolutions[composite_key] = (evolution_info) + self.traverse_and_store(next_stage, evolutions, gender) + def parse_evolution_chain(self, table, pokemon_form, force_refresh = False): - cache_record_name = f"evo_{pokemon_form["pfic"]}" + cache_record_name = f"evo_{pokemon_form['pfic']}" if force_refresh: cache.purge(cache_record_name) cached_entry = cache.get(cache_record_name) - if cached_entry != None: + if cached_entry is not None: return cached_entry - - main_chain = [] - current_stage = None - pending_method = None + form = get_form_name(pokemon_form, not pokemon_form["gender_relevant"]) tbody = table.find('tbody', recursive=False) if not tbody: - return [] + return None rows = tbody.find_all('tr', recursive=False) main_row = rows[0] branch_rows = rows[1:] + def create_stage(td): + pokemon_name = self.extract_pokemon_name(td) + evolution_form = self.extract_evolution_form(td, pokemon_name) + return { + "pokemon": pokemon_name, + "form": evolution_form, + "method": None, + "evolves_to": [] + } + # Parse main evolution chain + pending_method = None + root = None + current_stage = None + for td in main_row.find_all('td', recursive=False): if td.find('table'): - # This TD contains Pokemon information - pokemon_name = self.extract_pokemon_name(td) - stage = self.extract_stage_form(td) - evolution_form = self.extract_evolution_form(td, pokemon_name) - new_stage = { - "pokemon":pokemon_name, - "method": pending_method, - "stage": stage, - "form": evolution_form, - "next_stage": None, - "previous_stage": None, - "branches": [], - "pfic": pokemon_form["pfic"] - } + new_stage = create_stage(td) + new_stage["method"] = pending_method pending_method = None + if root is None: + root = new_stage # Assign the root node if current_stage: - current_stage["next_stage"] = new_stage - new_stage["previous_stage"] = current_stage # Set the back link + current_stage["evolves_to"].append(new_stage) current_stage = new_stage - main_chain.append(current_stage) else: - # This TD contains evolution method for the next Pokemon pending_method = self.extract_evolution_method(td) # Parse branching evolutions for row in branch_rows: - branch_stage = None branch_method = None + branch_stage = None + for td in row.find_all('td', recursive=False): if td.find('table'): - pokemon_name = self.extract_pokemon_name(td) - stage = self.extract_stage_form(td) - evolution_form = self.extract_evolution_form(td, pokemon_name) - new_stage = { - "pokemon":pokemon_name, - "method": branch_method, - "stage": stage, - "form": evolution_form, - "next_stage": None, - "previous_stage": None, - "branches": [], - "pfic": pokemon_form["pfic"] - } + new_stage = create_stage(td) + new_stage["method"] = branch_method branch_method = None if branch_stage: - branch_stage["next_stage"] = new_stage - new_stage["previous_stage"] = branch_stage # Set the back link + branch_stage["evolves_to"].append(new_stage) branch_stage = new_stage - # Find which main chain Pokemon this branches from - for main_stage in main_chain: - if td.get('rowspan') and main_stage.pokemon == pokemon_name: - main_stage["branches"].append(branch_stage) - branch_stage["previous_stage"] = main_stage # Set the back link to the main chain + + # Find which main chain Pokémon this branches from + for main_stage in self.find_stages(root): + if td.get('rowspan') and main_stage["pokemon"] == new_stage["pokemon"]: + main_stage["evolves_to"].append(branch_stage) break else: branch_method = self.extract_evolution_method(td) - cache.set(cache_record_name, main_chain) - return main_chain + cache.set(cache_record_name, root) + return root + + def find_stages(self, node): + """Helper function to find all stages in the evolution chain recursively.""" + stages = [node] + for stage in node["evolves_to"]: + stages.extend(self.find_stages(stage)) + return stages def extract_pokemon_name(self, td: Tag) -> Optional[str]: name_tag = self.find_name_tag(td) @@ -254,6 +275,16 @@ class GatherEvolutions(QRunnable): tbody = table.find('tbody', recursive=False) if not tbody: return [] + + def create_stage(td): + pokemon_name = self.extract_pokemon_name(td) + stage = self.extract_stage_form(td) + return { + "pokemon": pokemon_name, + "form": None, + "method": None, + "evolves_to": [] + } rows = tbody.find_all('tr', recursive=False) eevee_row = rows[1] @@ -261,17 +292,18 @@ class GatherEvolutions(QRunnable): eeveelutions_row = rows[3] eevee_td = eevee_row.find('td', recursive=False) - pokemon_name, stage = self.parse_pokemon_subtable(eevee_td) - eevee_stage = { - "pokemon":pokemon_name, - "method": None, - "stage": stage, - "form": None, - "next_stage": None, - "previous_stage": None, - "branches": [], - "pfic": pokemon_form["pfic"] - } + eevee_stage = create_stage(eevee_td) + #pokemon_name, stage = self.parse_pokemon_subtable(eevee_td) + #eevee_stage = { + # "pokemon":pokemon_name, + # "method": None, + # "stage": stage, + # "form": None, + # "next_stage": None, + # "previous_stage": None, + # "branches": [], + # "pfic": pokemon_form["pfic"] + #} methods = [] for method in method_row.find_all('td', recursive=False): @@ -280,24 +312,25 @@ class GatherEvolutions(QRunnable): eeveelutions = [] index = 0 for eeveelution in eeveelutions_row.find_all('td', recursive=False): - pokemon_name, stage = self.parse_pokemon_subtable(eeveelution) - eeveelution_stage = { - "pokemon":pokemon_name, - "method": methods[index], - "stage": stage, - "form": None, - "next_stage": None, - "previous_stage": None, - "branches": [], - "pfic": pokemon_form["pfic"] - } - eeveelution_stage["previous_stage"] = eevee_stage # Set the back link to Eevee + #pokemon_name, stage = self.parse_pokemon_subtable(eeveelution) + #eeveelution_stage = { + # "pokemon":pokemon_name, + # "method": methods[index], + # "stage": stage, + # "form": None, + # "next_stage": None, + # "previous_stage": None, + # "branches": [], + # "pfic": pokemon_form["pfic"] + #} + eeveelution_stage = create_stage(eeveelution) + #eeveelution_stage["previous_stage"] = eevee_stage # Set the back link to Eevee eeveelutions.append(eeveelution_stage) index += 1 - eevee_stage["branches"] = eeveelutions # Set the branches directly, not as a nested list + eevee_stage["evolves_to"] = eeveelutions # Set the branches directly, not as a nested list - return [eevee_stage] + return eevee_stage def parse_pokemon_subtable(self, td): if td.find('table'): diff --git a/utility/data.py b/utility/data.py index b53f354..f15e14c 100644 --- a/utility/data.py +++ b/utility/data.py @@ -302,4 +302,10 @@ default_forms = [ "Phony Form", "Masterpiece Form", "Chest Form" +] + +non_evolution_forms = [ + "Mega", + "Dynamax", + "Gigantamax" ] \ No newline at end of file -- 2.30.2 From 86ed6fb9edb09d2eb90233aad812e8ae3abd8510 Mon Sep 17 00:00:00 2001 From: Dan Date: Wed, 6 Nov 2024 11:06:43 +0000 Subject: [PATCH 09/21] - Notes to hopefully improve evolution determination --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index f4221eb..76149c6 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,8 @@ # OriginDex-DataManager +## Notes: + - Look into generating evolutions by just looking backwards from the current subject. + - Builbasaur sees nothing, it doesn't evolve from anything + - Ivysaur sees Bulbasaur + - Venusaur sees Ivysaur + - This should help deal with lines that have branching evolutions determines by gender, like Gallade and shouldn't impact the other less complex evolutions. \ No newline at end of file -- 2.30.2 From f7cfa89287b84e3a07677dd30ef3078a8c0bbb81 Mon Sep 17 00:00:00 2001 From: Quildra Date: Wed, 6 Nov 2024 22:00:25 +0000 Subject: [PATCH 10/21] WIP new way of extracting the node an edge work for evolutions --- database/db_controller.py | 31 ++++---- ui/workers/gather_evolutions_worker.py | 106 ++++++++++++------------- 2 files changed, 67 insertions(+), 70 deletions(-) diff --git a/database/db_controller.py b/database/db_controller.py index 0eca676..246be21 100644 --- a/database/db_controller.py +++ b/database/db_controller.py @@ -164,10 +164,11 @@ class DBController: self.conn.commit() def update_evolution_graph(self, evolutions): - for evolution in evolutions: - from_pfic = evolution["from_pfic"] - to_pfic = evolution["to_pfic"] - method = evolution["method"] + for key in evolutions: + value = evolutions[key] + from_pfic = value["from_pfic"] + to_pfic = value["to_pfic"] + method = value["method"] # Add nodes if they do not already exist if not self.graph.has_node(from_pfic): @@ -184,15 +185,15 @@ class DBController: def get_evolution_paths(self, start_node): paths = [] - + # Define a recursive function to traverse the graph def traverse(current_node, current_path): - # Add the current node to the path - current_path.append(current_node) - + # Add the current node to the path as a tuple (node, None) + current_path.append((current_node, None)) + # Get successors of the current node successors = list(self.graph.successors(current_node)) - + if not successors: # If there are no successors, add the current path to paths list paths.append(current_path.copy()) @@ -200,17 +201,19 @@ class DBController: # Traverse each successor and add edge metadata for successor in successors: method = self.graph[current_node][successor]["method"] - # Add the edge metadata as a tuple (to_node, method) + # Add the successor node and method as a tuple (successor, method) current_path.append((successor, method)) - + # Recur for the successor traverse(successor, current_path) - + # Backtrack (remove the last node and edge metadata) current_path.pop() - current_path.pop() + + # Remove the initial node tuple when backtracking fully + current_path.pop() # Start traversal from the start_node traverse(start_node, []) - + return paths diff --git a/ui/workers/gather_evolutions_worker.py b/ui/workers/gather_evolutions_worker.py index 712e424..5743064 100644 --- a/ui/workers/gather_evolutions_worker.py +++ b/ui/workers/gather_evolutions_worker.py @@ -11,7 +11,7 @@ from utility.functions import get_form_name, get_display_name, parse_pfic from utility.data import non_evolution_forms class GatherEvolutionsWorkerSignals(QObject): - finished = pyqtSignal(list) + finished = pyqtSignal(dict) class GatherEvolutions(QRunnable): def __init__(self): @@ -19,6 +19,8 @@ class GatherEvolutions(QRunnable): self.signals = GatherEvolutionsWorkerSignals() self.base_url = "https://bulbapedia.bulbagarden.net/wiki/" + self.evolution_methods = set() + def run(self): try: gathered_data = self.gather_evolution_data() @@ -28,7 +30,6 @@ class GatherEvolutions(QRunnable): def gather_evolution_data(self, force_refresh = True): all_pokemon_forms = db.get_list_of_pokemon_forms() - #evolutions = [] evolutions = {} for pokemon_form in all_pokemon_forms: @@ -90,55 +91,15 @@ class GatherEvolutions(QRunnable): evolution_tree = None if pokemon_name == "Eevee": evolution_tree = self.parse_eevee_evolution_chain(evolution_table, pokemon_form) - #evolutions.append(evolution_chain) else: evolution_tree = self.parse_evolution_chain(evolution_table, pokemon_form) - #evolutions.append(evolution_chain) if evolution_tree: self.traverse_and_store(evolution_tree, evolutions, gender) - chain = [] - for pokemon in evolution_chain: - from_pfic = self.get_pokemon_form_by_name(pokemon["pokemon"], pokemon["form"], gender=gender) - if not from_pfic: - #logger.warning(f"Could not find PFIC for {stage.pokemon} {stage.form}") - continue - - stage = pokemon["next_stage"] - if stage: - to_pfic = self.get_pokemon_form_by_name(stage["pokemon"], stage["form"], gender=gender) - if to_pfic: - evolution_info = { - "from_pfic": from_pfic, - "to_pfic": to_pfic, - "method": stage["method"] - } - evolutions[pokemon_form["pfic"]] = evolution_info - chain.append(evolution_info) - - #insert_evolution_info(evolution_info) - - #if "breed" in stage["next_stage"]["method"].lower(): - # update_pokemon_baby_status(from_pfic, True) - - for branch in pokemon["branches"]: - to_pfic = self.get_pokemon_form_by_name(branch["pokemon"], branch["form"], gender=gender) - if to_pfic: - evolution_info = { - "from_pfic": from_pfic, - "to_pfic": to_pfic, - "method": branch["method"] - } - evolutions[pokemon_form["pfic"]] = evolution_info - chain.append(evolution_info) - #EvolutionInfo(from_pfic, to_pfic, branch.method) - #insert_evolution_info(evolution_info) - - #if "breed" in branch.method.lower(): - # update_pokemon_baby_status(from_pfic, True) - cache.set(cache_record_name, chain) + #cache.set(cache_record_name, chain) + print(self.evolution_methods) return evolutions def traverse_and_store(self, node, evolutions, gender): @@ -159,7 +120,7 @@ class GatherEvolutions(QRunnable): evolutions[composite_key] = (evolution_info) self.traverse_and_store(next_stage, evolutions, gender) - def parse_evolution_chain(self, table, pokemon_form, force_refresh = False): + def parse_evolution_chain(self, table, pokemon_form, force_refresh = True): cache_record_name = f"evo_{pokemon_form['pfic']}" if force_refresh: cache.purge(cache_record_name) @@ -178,59 +139,85 @@ class GatherEvolutions(QRunnable): main_row = rows[0] branch_rows = rows[1:] - def create_stage(td): + def create_stage(td, current_stage_number): pokemon_name = self.extract_pokemon_name(td) evolution_form = self.extract_evolution_form(td, pokemon_name) return { "pokemon": pokemon_name, "form": evolution_form, + "requirement": None, "method": None, - "evolves_to": [] + "evolves_to": [], + "stage": current_stage_number } # Parse main evolution chain pending_method = None + pending_method_form = None root = None current_stage = None + stage_number = 0 for td in main_row.find_all('td', recursive=False): if td.find('table'): - new_stage = create_stage(td) + new_stage = create_stage(td, stage_number) new_stage["method"] = pending_method + new_stage["requirement"] = pending_method_form pending_method = None if root is None: root = new_stage # Assign the root node if current_stage: current_stage["evolves_to"].append(new_stage) current_stage = new_stage + stage_number += 1 else: - pending_method = self.extract_evolution_method(td) + pending_method, pending_method_form = self.extract_evolution_method(td) + + # reduce by one to account for an accidental increase by the last one in the chain. + stage_number -= 1 # Parse branching evolutions for row in branch_rows: branch_method = None + pending_method_form = None branch_stage = None + branch_stage_number = stage_number for td in row.find_all('td', recursive=False): if td.find('table'): - new_stage = create_stage(td) + new_stage = create_stage(td, branch_stage_number) new_stage["method"] = branch_method + new_stage["requirement"] = pending_method_form branch_method = None + if branch_stage: branch_stage["evolves_to"].append(new_stage) branch_stage = new_stage - # Find which main chain Pokémon this branches from + # Find which main chain Pokémon this branch evolves from + attached = False for main_stage in self.find_stages(root): - if td.get('rowspan') and main_stage["pokemon"] == new_stage["pokemon"]: + if self.should_attach_branch(main_stage, branch_stage): main_stage["evolves_to"].append(branch_stage) + attached = True break + + if not attached: + print(f"Warning: Could not find a suitable attachment point for branch {branch_stage['pokemon']}") else: - branch_method = self.extract_evolution_method(td) + branch_method, pending_method_form = self.extract_evolution_method(td) cache.set(cache_record_name, root) return root + def should_attach_branch(self, main_stage, branch_stage): + # Ensure the main_stage is a valid node to attach a branch + if main_stage["stage"] == branch_stage["stage"] - 1: + return True + # You can add more logic to determine if branch_stage should connect to main_stage + # For instance, check if they are forms of the same evolution or based on other criteria + return False + def find_stages(self, node): """Helper function to find all stages in the evolution chain recursively.""" stages = [node] @@ -269,7 +256,14 @@ class GatherEvolutions(QRunnable): def extract_evolution_method(self, td: Tag) -> str: # Extract evolution method from the TD - return td.get_text(strip=True) + text = td.get_text() + form = None + if text and "(male)" in text.lower(): + form = "male" + elif text and "(female)" in text.lower(): + form = "female" + + return td.get_text(strip=True), form def parse_eevee_evolution_chain(self, table, pokemon_form): tbody = table.find('tbody', recursive=False) @@ -376,8 +370,8 @@ class GatherEvolutions(QRunnable): return entry["pfic"] # Some times we get a form for a pokemon that doesn't really have one. - if len(results) > 1 and form != None: - return results[0]["pfic"] + #if len(results) > 1 and form != None and gender and threshold != 100: + # return results[0]["pfic"] return None -- 2.30.2 From 107422679b1a257ff25be0a7a1f9e8d7273e8f5e Mon Sep 17 00:00:00 2001 From: Dan Date: Thu, 7 Nov 2024 14:07:43 +0000 Subject: [PATCH 11/21] - A good place for evolutions now. They are tracking in a node graph, and displayed correctly. --- database/db_controller.py | 110 +++++++++++++++++++++++-- ui/main_window_controller.py | 3 +- ui/main_window_view.py | 93 ++++++++++++++------- ui/workers/gather_evolutions_worker.py | 21 ++--- 4 files changed, 179 insertions(+), 48 deletions(-) diff --git a/database/db_controller.py b/database/db_controller.py index 246be21..62c4d32 100644 --- a/database/db_controller.py +++ b/database/db_controller.py @@ -181,15 +181,34 @@ class DBController: self.graph.add_edge(from_pfic, to_pfic, method=method) def get_evolution_graph(self, pfic): + if self.graph.has_node(pfic) == False: + return [] + return list(self.graph.successors(pfic)) + def get_previous_evolution(self, pfic): + if self.graph.has_node(pfic) == False: + return None, None + + predecessor = next(self.graph.predecessors(pfic), None) + + if predecessor: + method = self.graph[predecessor][pfic]["method"] + return predecessor, method + else: + return None, None + def get_evolution_paths(self, start_node): paths = [] + if self.graph.has_node(start_node) == False: + return paths + # Define a recursive function to traverse the graph - def traverse(current_node, current_path): - # Add the current node to the path as a tuple (node, None) - current_path.append((current_node, None)) + def traverse(current_node, current_path, is_root=False): + if is_root: + # Add the current node to the path as a tuple (node, None) + current_path.append((current_node, None)) # Get successors of the current node successors = list(self.graph.successors(current_node)) @@ -211,9 +230,90 @@ class DBController: current_path.pop() # Remove the initial node tuple when backtracking fully - current_path.pop() + if is_root: + current_path.pop() # Start traversal from the start_node - traverse(start_node, []) + traverse(start_node, [], True) return paths + + def get_full_evolution_paths(self, start_node): + """ + Get all evolution paths starting from a given node, including predecessors and successors. + :param start_node: The starting node (e.g., a specific Pokemon form). + :return: A dictionary containing predecessors and successors paths. + """ + full_paths = { + "predecessors": [], + "successors": [] + } + + if self.graph.has_node(start_node) == False: + return full_paths + + # Traverse predecessors + def traverse_predecessors(current_node, current_path, is_root=False): + #if not is_root: + # Add the current node to the path + #current_path.append(current_node) + + # Get predecessors of the current node + predecessors = list(self.graph.predecessors(current_node)) + + if not predecessors: + # If there are no predecessors, add the current path to the list + full_paths["predecessors"].append(current_path.copy()) + else: + # Traverse each predecessor + for predecessor in predecessors: + method = self.graph[predecessor][current_node]["method"] + # Add the edge metadata as a tuple (predecessor, method) + current_path.append((predecessor, method)) + + # Recur for the predecessor + traverse_predecessors(predecessor, current_path) + + # Backtrack (remove the last node and edge metadata) + current_path.pop() + #current_path.pop() + + # Traverse successors + def traverse_successors(current_node, current_path, is_root=False): + if is_root: + # Add the current node to the path as a tuple (node, None) + predecessor = next(self.graph.predecessors(current_node), None) + if predecessor: + method = self.graph[predecessor][current_node]["method"] + current_path.append((current_node, method)) + else: + current_path.append((current_node, None)) + + # Get successors of the current node + successors = list(self.graph.successors(current_node)) + + if not successors: + # If there are no successors, add the current path to paths list + full_paths["successors"].append(current_path.copy()) + else: + # Traverse each successor and add edge metadata + for successor in successors: + method = self.graph[current_node][successor]["method"] + # Add the successor node and method as a tuple (successor, method) + current_path.append((successor, method)) + + # Recur for the successor + traverse_successors(successor, current_path) + + # Backtrack (remove the last node and edge metadata) + current_path.pop() + + if is_root: + # Remove the initial node tuple when backtracking fully + current_path.pop() + + # Start traversal from the start_node for both predecessors and successors + traverse_predecessors(start_node, [], True) + traverse_successors(start_node, [], True) + + return full_paths \ No newline at end of file diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index 8723d36..33099fb 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -174,5 +174,6 @@ class MainWindowController: self.current_pfic = pfic def load_evolution_chain(self, pfic): - chain = db.get_evolution_paths(pfic) + #chain = db.get_evolution_paths(pfic) + chain = db.get_full_evolution_paths(pfic) self.view.update_evolution_tree(chain, pfic) diff --git a/ui/main_window_view.py b/ui/main_window_view.py index fafdc6c..760f1ad 100644 --- a/ui/main_window_view.py +++ b/ui/main_window_view.py @@ -223,41 +223,70 @@ class PokemonUI(QWidget): item.setData(Qt.ItemDataRole.UserRole, pokemon["pfic"]) self.pokemon_list.addItem(item) - def update_evolution_tree(self, evolution_chain, selected_pfic): + def update_evolution_tree(self, evolution_chains, selected_pfic): + self.evolution_tree.clear() tree_items = {} - #for item in evolution_chain: - # print(item) - for pfic in evolution_chain: - pokemon_details = db.get_pokemon_details(pfic) - display_name = get_display_name(pokemon_details, not pokemon_details["gender_relevant"]) - item = QTreeWidgetItem([display_name, method if method else ""]) - item.setData(0, Qt.ItemDataRole.UserRole, current_pfic) - tree_items[current_pfic] = item - - if current_pfic == selected_pfic: - item.setBackground(0, QColor(255, 255, 0, 100)) # Highlight selected Pokémon - - # Second pass: build the tree structure - root = None - for current_pfic, name, form_name, method in evolution_chain: - item = tree_items[current_pfic] - - # Find the parent of this item - #parent_pfic = event_system.call_sync('get_evolution_parent', data=current_pfic) - parent_pfic = None - - if parent_pfic: - parent_item = tree_items.get(parent_pfic[0]) - if parent_item: - parent_item.addChild(item) - elif not root: - root = item - self.evolution_tree.addTopLevelItem(root) + + for chains in evolution_chains["predecessors"]: + for pfic, method in chains: + pokemon_details = db.get_pokemon_details(pfic) + display_name = get_display_name(pokemon_details, not pokemon_details["gender_relevant"]) + item = QTreeWidgetItem([display_name, method if method else ""]) + item.setData(0, Qt.ItemDataRole.UserRole, pfic) + tree_items[pfic] = item + + if pfic == selected_pfic: + item.setBackground(0, QColor(255, 255, 0, 100)) # Highlight selected Pokémon + + # Second pass: build the tree structure + root = None + for pfic, method in chains: + item = tree_items[pfic] + + # Find the parent of this item + parent_pfic, method = db.get_previous_evolution(pfic) + + if parent_pfic: + parent_item = tree_items.get(parent_pfic) + if parent_item: + parent_item.addChild(item) + elif not root: + root = item + self.evolution_tree.addTopLevelItem(root) + + for chains in evolution_chains["successors"]: + for pfic, method in chains: + pokemon_details = db.get_pokemon_details(pfic) + display_name = get_display_name(pokemon_details, not pokemon_details["gender_relevant"]) + item = QTreeWidgetItem([display_name, method if method else ""]) + item.setData(0, Qt.ItemDataRole.UserRole, pfic) + tree_items[pfic] = item + + if pfic == selected_pfic: + item.setBackground(0, QColor(255, 255, 0, 100)) # Highlight selected Pokémon + + # Second pass: build the tree structure + root = None + for pfic, method in chains: + item = tree_items[pfic] + + # Find the parent of this item + parent_pfic, method = db.get_previous_evolution(pfic) + + if parent_pfic: + parent_item = tree_items.get(parent_pfic) + if parent_item: + parent_item.addChild(item) + elif not root: + root = item + self.evolution_tree.addTopLevelItem(root) + # Expand the entire tree self.evolution_tree.expandAll() # Scroll to and select the current Pokémon - current_item = tree_items[selected_pfic] - self.evolution_tree.scrollToItem(current_item) - self.evolution_tree.setCurrentItem(current_item) \ No newline at end of file + if selected_pfic in tree_items: + current_item = tree_items[selected_pfic] + self.evolution_tree.scrollToItem(current_item) + self.evolution_tree.setCurrentItem(current_item) \ No newline at end of file diff --git a/ui/workers/gather_evolutions_worker.py b/ui/workers/gather_evolutions_worker.py index 5743064..22d9c05 100644 --- a/ui/workers/gather_evolutions_worker.py +++ b/ui/workers/gather_evolutions_worker.py @@ -28,7 +28,7 @@ class GatherEvolutions(QRunnable): except Exception as e: print(f"Error gathering Pokémon home storage status: {e}") - def gather_evolution_data(self, force_refresh = True): + def gather_evolution_data(self, force_refresh = False): all_pokemon_forms = db.get_list_of_pokemon_forms() evolutions = {} @@ -44,10 +44,10 @@ class GatherEvolutions(QRunnable): if force_refresh: cache.purge(cache_record_name) - #cached_entry = cache.get(cache_record_name) - #if cached_entry != None: - # evolutions[pokemon_form["pfic"]] = cached_entry - # continue + cached_entry = cache.get(cache_record_name) + if cached_entry != None: + evolutions = evolutions | cached_entry + continue #form = get_form_name(pokemon_form, not pokemon_form["gender_relevant"]) search_form = form @@ -85,19 +85,20 @@ class GatherEvolutions(QRunnable): if tag.name == 'h3': break if not evolution_table: - continue + continue - evolution_chain = [] evolution_tree = None if pokemon_name == "Eevee": evolution_tree = self.parse_eevee_evolution_chain(evolution_table, pokemon_form) else: evolution_tree = self.parse_evolution_chain(evolution_table, pokemon_form) + cacheable_container = {} if evolution_tree: - self.traverse_and_store(evolution_tree, evolutions, gender) + self.traverse_and_store(evolution_tree, cacheable_container, gender) - #cache.set(cache_record_name, chain) + cache.set(cache_record_name, cacheable_container) + evolutions = evolutions | cacheable_container print(self.evolution_methods) return evolutions @@ -120,7 +121,7 @@ class GatherEvolutions(QRunnable): evolutions[composite_key] = (evolution_info) self.traverse_and_store(next_stage, evolutions, gender) - def parse_evolution_chain(self, table, pokemon_form, force_refresh = True): + def parse_evolution_chain(self, table, pokemon_form, force_refresh = False): cache_record_name = f"evo_{pokemon_form['pfic']}" if force_refresh: cache.purge(cache_record_name) -- 2.30.2 From 69466079f0fab34448cf3576fd748b7e28b481b7 Mon Sep 17 00:00:00 2001 From: Dan Date: Thu, 7 Nov 2024 15:33:02 +0000 Subject: [PATCH 12/21] - Added a pass to set up gender relevance depening on form or evolution --- database/db_controller.py | 50 +++++++++++++++++++++++++++++++++++- ui/main_window_controller.py | 9 +++++++ ui/main_window_view.py | 4 +++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/database/db_controller.py b/database/db_controller.py index 62c4d32..e176510 100644 --- a/database/db_controller.py +++ b/database/db_controller.py @@ -316,4 +316,52 @@ class DBController: traverse_predecessors(start_node, [], True) traverse_successors(start_node, [], True) - return full_paths \ No newline at end of file + return full_paths + + def propagate_gender_relevance(self, gender_relevant_nodes): + """ + Propagate gender relevance through the evolution graph and update the SQLite database. + :param db_path: Path to the SQLite database file. + :param gender_relevant_nodes: A set of nodes that are initially marked as gender-relevant. + """ + + # Traverse from each gender-relevant end node backward to propagate relevance + for node in gender_relevant_nodes: + # Use breadth-first search or depth-first search to traverse backward + visited = set() + stack = [node] + + while stack: + current_node = stack.pop() + if current_node not in visited: + visited.add(current_node) + + # Update the gender_relevant flag in the database + self.update_pokemon_field(current_node, "gender_relevant", True) + + # Add predecessors to the stack to keep traversing backward + if self.graph.has_node(current_node): + predecessors = list(self.graph.predecessors(current_node)) + stack.extend(predecessors) + + self.save_changes() + + def get_gender_specific_evolutions(self): + """ + Get a list of nodes that have evolution methods indicating gender relevance (i.e., '(male)' or '(female)'). + :return: A list of nodes involved in gender-specific evolutions. + """ + gender_specific_nodes = [] + + for from_node, to_node, edge_data in self.graph.edges(data=True): + method = edge_data.get("method", "") + if method and ("(male)" in method.lower() or "(female)" in method.lower()): + # Add both nodes involved in this gender-specific evolution + gender_specific_nodes.extend([from_node, to_node]) + + return list(set(gender_specific_nodes)) # Return unique nodes + + def get_gender_relevant_pokemon(self): + self.cursor.execute(f"SELECT PFIC FROM pokemon_forms WHERE JSON_EXTRACT(data, '$.gender_relevant') = true") + results = self.cursor.fetchall() + return [row['PFIC'] for row in results] \ No newline at end of file diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index 33099fb..12cd29a 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -127,6 +127,15 @@ class MainWindowController: print("Works Done!") db.update_evolution_graph(data) + def adjust_gender_relevancy(self): + list = db.get_gender_specific_evolutions() + second_list = db.get_gender_relevant_pokemon() + print(list) + print(second_list) + db.propagate_gender_relevance(list) + db.propagate_gender_relevance(second_list) + pass + def reinitialize_database(self): pass diff --git a/ui/main_window_view.py b/ui/main_window_view.py index 760f1ad..c2968a2 100644 --- a/ui/main_window_view.py +++ b/ui/main_window_view.py @@ -152,6 +152,10 @@ class PokemonUI(QWidget): gather_evolutions_btn.clicked.connect(self.controller.gather_evolution_info) db_tab_layout.addWidget(gather_evolutions_btn) + gather_evolutions_btn = QPushButton("Adjust Gender Relevant Information") + gather_evolutions_btn.clicked.connect(self.controller.adjust_gender_relevancy) + db_tab_layout.addWidget(gather_evolutions_btn) + gather_encounters_btn = QPushButton("Gather Encounter Information") gather_encounters_btn.clicked.connect(self.controller.gather_encounter_info) db_tab_layout.addWidget(gather_encounters_btn) -- 2.30.2 From 35c14c9a57565a07bebfcedeb590c714a6386a42 Mon Sep 17 00:00:00 2001 From: Dan Date: Thu, 7 Nov 2024 16:05:23 +0000 Subject: [PATCH 13/21] - Fix up the storable in home filter --- database/db_controller.py | 23 ++++++++++--------- ui/main_window_controller.py | 7 +++--- .../gather_home_storage_status_worker.py | 7 ++++++ 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/database/db_controller.py b/database/db_controller.py index e176510..6d28b4c 100644 --- a/database/db_controller.py +++ b/database/db_controller.py @@ -99,16 +99,17 @@ class DBController: return query - def get_pokemon_details(self, pfic): - fields = [ - "name", - "form_name", - "national_dex", - "generation", - "is_baby_form", - "storable_in_home", - "gender_relevant" - ] + def get_pokemon_details(self, pfic, fields = None): + if fields == None: + fields = [ + "name", + "form_name", + "national_dex", + "generation", + "is_baby_form", + "storable_in_home", + "gender_relevant" + ] query = self.craft_pokemon_json_query(fields, pfic) self.cursor.execute(query) results = self.cursor.fetchone() @@ -360,7 +361,7 @@ class DBController: gender_specific_nodes.extend([from_node, to_node]) return list(set(gender_specific_nodes)) # Return unique nodes - + def get_gender_relevant_pokemon(self): self.cursor.execute(f"SELECT PFIC FROM pokemon_forms WHERE JSON_EXTRACT(data, '$.gender_relevant') = true") results = self.cursor.fetchall() diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index 12cd29a..53240c6 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -46,6 +46,7 @@ class MainWindowController: if show_only_home_storable: # TODO: update the call to correctly filter the data, or better yet update the data at the source to include this info. home_storable = True #event_system.call_sync('get_home_storable', pfic) + home_storable = db.get_pokemon_details(pfic, ["storable_in_home"])["storable_in_home"] # Check to see if the pokemon has encounters has_encounters = True @@ -74,8 +75,9 @@ class MainWindowController: context_menu.exec(self.pokemon_list.viewport().mapToGlobal(position)) def on_pokemon_selected(self, item): - pfic = item.data(Qt.ItemDataRole.UserRole) - self.refresh_pokemon_details_panel(pfic) + if item: + pfic = item.data(Qt.ItemDataRole.UserRole) + self.refresh_pokemon_details_panel(pfic) def edit_encounter(self): pass @@ -183,6 +185,5 @@ class MainWindowController: self.current_pfic = pfic def load_evolution_chain(self, pfic): - #chain = db.get_evolution_paths(pfic) chain = db.get_full_evolution_paths(pfic) self.view.update_evolution_tree(chain, pfic) diff --git a/ui/workers/gather_home_storage_status_worker.py b/ui/workers/gather_home_storage_status_worker.py index 85351d0..e63846a 100644 --- a/ui/workers/gather_home_storage_status_worker.py +++ b/ui/workers/gather_home_storage_status_worker.py @@ -49,6 +49,10 @@ class GatherHomeStorageStatus(QRunnable): if working_form and name in working_form: working_form = working_form.replace(name, "").strip() + if working_form: + working_form = working_form.replace("Female", "").replace("female", "").strip() + working_form = working_form.replace("Male", "").replace("male", "").strip() + # serebii doesn't list gender in the table so we have to assume based on form name. if working_form and ("male" in working_form.lower() or "working_form" in working_form.lower()): working_form = None @@ -63,6 +67,9 @@ class GatherHomeStorageStatus(QRunnable): if name == "Alcremie": working_form = None + if working_form == "": + working_form = None + for pokemon in pokemon_by_national_dex[national_dex]: if working_form: parts = pokemon['name'].split(" ") -- 2.30.2 From e5f109949a9416fa85017dc58ebc1995da711f01 Mon Sep 17 00:00:00 2001 From: Dan Date: Fri, 8 Nov 2024 16:06:35 +0000 Subject: [PATCH 14/21] - WIP reworking form p-arsing to be more consistant --- ui/main_window_controller.py | 1 + ui/workers/gather_evolutions_worker.py | 4 +- .../gather_home_storage_status_worker.py | 2 +- ui/workers/gather_pokemon_forms_worker.py | 4 ++ utility/data.py | 16 ++++-- utility/pokemon_word_ninja.py | 54 +++++++++++++++++++ 6 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 utility/pokemon_word_ninja.py diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index 53240c6..01056e8 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -106,6 +106,7 @@ class MainWindowController: db.add_pokemon_form(pokemon["pfic"], pokemon["name"], pokemon["form_name"], pokemon["national_dex"], pokemon["generation"], pokemon["sprite_url"], pokemon["gender_relevant"]) self.pokemon_data_cache = data self.view.update_pokemon_forms(data) + self.apply_filters() db.save_changes() diff --git a/ui/workers/gather_evolutions_worker.py b/ui/workers/gather_evolutions_worker.py index 22d9c05..9dfbd11 100644 --- a/ui/workers/gather_evolutions_worker.py +++ b/ui/workers/gather_evolutions_worker.py @@ -28,7 +28,7 @@ class GatherEvolutions(QRunnable): except Exception as e: print(f"Error gathering Pokémon home storage status: {e}") - def gather_evolution_data(self, force_refresh = False): + def gather_evolution_data(self, force_refresh = True): all_pokemon_forms = db.get_list_of_pokemon_forms() evolutions = {} @@ -121,7 +121,7 @@ class GatherEvolutions(QRunnable): evolutions[composite_key] = (evolution_info) self.traverse_and_store(next_stage, evolutions, gender) - def parse_evolution_chain(self, table, pokemon_form, force_refresh = False): + def parse_evolution_chain(self, table, pokemon_form, force_refresh = True): cache_record_name = f"evo_{pokemon_form['pfic']}" if force_refresh: cache.purge(cache_record_name) diff --git a/ui/workers/gather_home_storage_status_worker.py b/ui/workers/gather_home_storage_status_worker.py index e63846a..7384a47 100644 --- a/ui/workers/gather_home_storage_status_worker.py +++ b/ui/workers/gather_home_storage_status_worker.py @@ -104,7 +104,7 @@ class GatherHomeStorageStatus(QRunnable): if cached_entry is not None: return cached_entry - url = f"{self.base_url}{region}pokemon.shtml" + url = f"{self.base_url}{region.lower()}pokemon.shtml" response = cache.fetch_url(url) if not response: return [] diff --git a/ui/workers/gather_pokemon_forms_worker.py b/ui/workers/gather_pokemon_forms_worker.py index e41fec6..fe2d690 100644 --- a/ui/workers/gather_pokemon_forms_worker.py +++ b/ui/workers/gather_pokemon_forms_worker.py @@ -4,6 +4,7 @@ import re from cache import cache from utility.functions import get_generation_from_national_dex, sanitise_pokemon_name_for_url, remove_accents, compare_pokemon_forms, find_game_generation, format_pokemon_id +from utility.pokemon_word_ninja import PokemonWordNinja class GatherPokemonFormsWorkerSignals(QObject): finished = pyqtSignal(list) @@ -12,6 +13,7 @@ class GatherPokemonFormsWorker(QRunnable): def __init__(self): super().__init__() self.signals = GatherPokemonFormsWorkerSignals() + self.splitter = PokemonWordNinja() def run(self): try: @@ -56,6 +58,7 @@ class GatherPokemonFormsWorker(QRunnable): for small in smalls: form_name += small.get_text(strip=True) + " " form_name = form_name.strip() + form_name = self.splitter.split(form_name) return form_name return "None" @@ -63,6 +66,7 @@ class GatherPokemonFormsWorker(QRunnable): found_forms = [] generation = get_generation_from_national_dex(national_dex_number) pokemon_name = pokemon_soup.get_text(strip=True) + self.splitter.add_custom_word(pokemon_name) print(f"Processing {pokemon_name}") url_name = sanitise_pokemon_name_for_url(pokemon_name) diff --git a/utility/data.py b/utility/data.py index f15e14c..c289958 100644 --- a/utility/data.py +++ b/utility/data.py @@ -10,8 +10,8 @@ pokemon_generations = { 9: {"min": 906, "max": 1025}, } -regions = ["kanto", "johto", "hoenn", "sinnoh", "unova", "kalos", "alola", "galar", "paldea", "hisui", "unknown"] -regional_descriptors = ["kantonian", "johtonian", "hoennian", "sinnohan", "unovan", "kalosian", "alolan", "galarian", "hisuian", "paldean"] +regions = ["Kanto", "Johto", "Hoenn", "Sinnoh", "Unova", "Kalos", "Alola", "Galar", "Paldea", "Hisui", "Unknown"] +regional_descriptors = ["Kantonian", "Johtonian", "Hoennian", "Sinnohan", "Unovan", "Kalosian", "Alolan", "Galarian", "Hisuian", "Paldean"] yellow = { "Name": "Yellow", @@ -308,4 +308,14 @@ non_evolution_forms = [ "Mega", "Dynamax", "Gigantamax" -] \ No newline at end of file +] + +POKEMON_PROPER_NOUNS = { + "Augurite", + "Electirizer", + "Magmarizer", + "Gigantamax" +} + +POKEMON_PROPER_NOUNS = POKEMON_PROPER_NOUNS | set(regions) +POKEMON_PROPER_NOUNS = POKEMON_PROPER_NOUNS | set(regional_descriptors) \ No newline at end of file diff --git a/utility/pokemon_word_ninja.py b/utility/pokemon_word_ninja.py new file mode 100644 index 0000000..2ff87f4 --- /dev/null +++ b/utility/pokemon_word_ninja.py @@ -0,0 +1,54 @@ +import wordninja +import re +from typing import List +from utility.data import POKEMON_PROPER_NOUNS + +class PokemonWordNinja: + def __init__(self, custom_word_list: List[str] = None): + custom_words = POKEMON_PROPER_NOUNS + if custom_word_list: + custom_words = custom_words | set(custom_word_list) + + self.custom_words = [] + self.placeholder_map = {} + self.word_to_placeholder_map = {} + if custom_words: + # Store custom words with original capitalization, sorted by length + self.custom_words = sorted(custom_words, key=len, reverse=True) + for word in self.custom_words: + # Generate a unique placeholder + placeholder = f"__PLACEHOLDER_{hash(word)}__" + self.placeholder_map[placeholder] = word + self.word_to_placeholder_map[word] = placeholder + + def add_custom_word(self, word: str): + words = self.custom_words + words.append(word) + self.custom_words = sorted(words, key=len, reverse=True) + placeholder = f"__PLACEHOLDER_{hash(word)}__" + self.placeholder_map[placeholder] = word + self.word_to_placeholder_map[word] = placeholder + + def split(self, text: str) -> str: + working_text = text + + # First handle exact custom words to preserve capitalization + for word in self.custom_words: + placeholder = self.word_to_placeholder_map[word] + pattern = re.compile(re.escape(word), re.IGNORECASE) + working_text = pattern.sub(placeholder, working_text) + + # Clean up spaces + working_text = ' '.join(working_text.split()) + + # For remaining text, use wordninja + parts = [] + for part in working_text.split(): + if part in self.placeholder_map: + # Replace placeholder with the original word + parts.append(self.placeholder_map[part]) + else: + split_parts = wordninja.split(part) + parts.extend(split_parts) + + return ' '.join(parts) \ No newline at end of file -- 2.30.2 From 8f88244cc38cedeeb2d2682514728b3347d0df2b Mon Sep 17 00:00:00 2001 From: Quildra Date: Sat, 9 Nov 2024 21:52:20 +0000 Subject: [PATCH 15/21] - Added in the encounter gathering --- database/db_controller.py | 101 +++- ui/main_window_controller.py | 9 +- ui/workers/gather_encounter_locations.py | 699 +++++++++++++++++++++++ utility/data.py | 6 +- utility/functions.py | 40 +- 5 files changed, 850 insertions(+), 5 deletions(-) create mode 100644 ui/workers/gather_encounter_locations.py diff --git a/database/db_controller.py b/database/db_controller.py index 6d28b4c..4542c95 100644 --- a/database/db_controller.py +++ b/database/db_controller.py @@ -4,6 +4,8 @@ import json import os import networkx as nx +from utility.data import main_line_games + class DBController: def __init__(self, db_path=':memory:', max_connections=10): self.db_path = db_path @@ -20,6 +22,8 @@ class DBController: # Create tables in the file-based database self.create_pokemon_forms_table(disk_cursor) + self.create_games_table(disk_cursor) + self.create_encounters_table(disk_cursor) # Commit changes to the file-based database disk_conn.commit() @@ -64,6 +68,39 @@ class DBController: ) ''') + def create_encounters_table(self, cursor): + cursor.execute(''' + CREATE TABLE IF NOT EXISTS encounters ( + PFIC TEXT, + game_id INTEGER NOT NULL, + type TEXT NOT NULL, + data JSON NOT NULL, + FOREIGN KEY (PFIC) REFERENCES pokemon_forms (PFIC), + FOREIGN KEY (game_id) REFERENCES games (id) + ) + ''') + + def create_games_table(self, cursor): + cursor.execute(''' + CREATE TABLE IF NOT EXISTS games ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + alt_names TEXT, + generation INTEGER NOT NULL, + data JSON + ) + ''') + + for game in main_line_games: + name = game["Name"] + alt_names = ", ".join(game["AltNames"]) # Convert list to comma-separated string + generation = game["Generation"] + + cursor.execute(''' + INSERT OR IGNORE INTO games (name, alt_names, generation) + VALUES (?, ?, ?) + ''', (name, alt_names, generation)) + def add_pokemon_form(self, pfic, name, form_name, national_dex, generation, sprite_url, gender_relevant): data = { "name": name, @@ -115,7 +152,18 @@ class DBController: results = self.cursor.fetchone() return dict(results) - def get_pokemon_details_by_name(self, name, fields): + def get_pokemon_details_by_name(self, name, fields = None): + if fields == None: + fields = [ + "pfic", + "name", + "form_name", + "national_dex", + "generation", + "is_baby_form", + "storable_in_home", + "gender_relevant" + ] query = self.craft_pokemon_json_query(fields) name = name.replace("'", "''") query += f" WHERE JSON_EXTRACT(data, '$.name') = '{name}'" @@ -365,4 +413,53 @@ class DBController: def get_gender_relevant_pokemon(self): self.cursor.execute(f"SELECT PFIC FROM pokemon_forms WHERE JSON_EXTRACT(data, '$.gender_relevant') = true") results = self.cursor.fetchall() - return [row['PFIC'] for row in results] \ No newline at end of file + return [row['PFIC'] for row in results] + + def get_game_id_by_name(self, name): + self.cursor.execute(''' + SELECT id, name, generation FROM games + WHERE name LIKE ? OR alt_names LIKE ? + ''', (f"%{name}%", f"%{name}%")) + + # Fetch and print the results + result = self.cursor.fetchone() + print(f"ID: {result[0]}, Name: {result[1]}, Generation: {result[2]}") + + return dict(result) + + def get_games_by_name(self, name): + self.cursor.execute(''' + SELECT id, name, generation FROM games + WHERE name LIKE ? OR alt_names LIKE ? + ''', (f"%{name}%", f"%{name}%")) + + # Fetch and print the results + results = self.cursor.fetchall() + return [dict(row) for row in results] + + def get_games_by_generation(self, generation): + self.cursor.execute(''' + SELECT id, name FROM games + WHERE generation = ? + ''', (generation,)) + + # Fetch and print the results + results = self.cursor.fetchall() + for row in results: + print(f"ID: {row[0]}, Name: {row[1]}") + + return [dict(row) for row in results] + + def update_encounter_locations(self, data): + for encounter in data: + with self.lock: + pfic = encounter["pfic"] + game_id = encounter["game_id"]["id"] + type = encounter["type"] + data = encounter["data"] if "data" in encounter else None + self.cursor.execute(''' + INSERT OR REPLACE INTO encounters (PFIC, game_id, type, data) VALUES (?, ?, ?, ?) + ''', (pfic, game_id, type, json.dumps(data))) + self.conn.commit() + print(f"Added: {pfic}") + pass \ No newline at end of file diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index 01056e8..379510a 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -3,6 +3,7 @@ from PyQt6.QtWidgets import QMenu from PyQt6.QtGui import QAction import os +from ui.workers.gather_encounter_locations import GatherEncountersWorker from ui.workers.gather_home_storage_status_worker import GatherHomeStorageStatus from ui.workers.gather_pokemon_forms_worker import GatherPokemonFormsWorker from ui.workers.gather_evolutions_worker import GatherEvolutions @@ -143,7 +144,13 @@ class MainWindowController: pass def gather_encounter_info(self): - pass + worker = GatherEncountersWorker() + worker.signals.finished.connect(self.on_encounters_gathered) + self.thread_pool.start(worker) + + def on_encounters_gathered(self, data): + print("Works Done!") + db.update_encounter_locations(data) def gather_marks_info(self): pass diff --git a/ui/workers/gather_encounter_locations.py b/ui/workers/gather_encounter_locations.py new file mode 100644 index 0000000..ca2159e --- /dev/null +++ b/ui/workers/gather_encounter_locations.py @@ -0,0 +1,699 @@ +from PyQt6.QtCore import QObject, pyqtSignal, QRunnable +from bs4 import BeautifulSoup, NavigableString +from pattern.en import singularize +from fuzzywuzzy import fuzz +import re + +from cache import cache +from db import db + +from utility.data import default_forms, regional_descriptors, days, times, rods +from utility.functions import is_mainline_game, compare_pokemon_forms, find_match_in_string_array, extract_bracketed_text + +class GatherEncountersWorkerSignals(QObject): + finished = pyqtSignal(list) + +class GatherEncountersWorker(QRunnable): + def __init__(self): + super().__init__() + self.signals = GatherEncountersWorkerSignals() + self.default_forms_set = set(default_forms) + self.encounters_to_ignore = [ + "trade", + "time capsule", + "unobtainable", + "tradeversion", + "poké transfer", + "friend safari", + "unavailable", + "pokémon home", + "union circle", + "pokémon bank", + "pal park", + "transfer from dream radar", + "global link event", + "pokémon channel", + "pokémon colosseum bonus disc" + ] + self.encounters = [] + + def run(self): + try: + gathered_data = self.gather_encounter_data() + self.signals.finished.emit(gathered_data) + except Exception as e: + print(f"Error gathering Pokémon forms: {e}") + + def gather_encounter_data(self): + all_pokemon_forms = db.get_list_of_pokemon_forms() + + for form_entry in all_pokemon_forms: + form = form_entry["form_name"] + name = form_entry["name"] + pfic = form_entry["pfic"] + + print(f'Processing {name}') + + if form and name in form: + form = form.replace(name, "").strip() + + if form and form in default_forms: + form = None + + if name == "Unown" and (form != "!" and form != "?"): + form = None + + if name == "Tauros" and form == "Combat Breed": + form = "Paldean Form" + + if name == "Alcremie": + form = None + + if name == "Minior": + form = None + + if name.lower() == "ho-oh": + name = "Ho-Oh" + + if form and form.startswith("Female"): + form = form.replace("Female", "").strip() + + if form and form.startswith("Male"): + form = form.replace("Male", "").strip() + + if form == "": + form = None + + search_form = form + + encounter_data = self.get_locations_from_bulbapedia(name, search_form) + if encounter_data == None: + continue + + for encounter in encounter_data: + if len(encounter_data[encounter]) == 0: + break + + for location in encounter_data[encounter]: + if location == "": + continue + test_location = location["location"].strip().lower() + test_location_text = BeautifulSoup(test_location, 'html.parser').get_text().lower() + if "evolve" in test_location_text: + remaining, details = self.extract_additional_information(location["tag"]) + evolve_info = self.extract_evolve_information(remaining) + + if evolve_info: + #logger.info(f"Evolve Info: {evolve_info}") + self.save_evolve_encounter(pfic, encounter, details["days"], details["times"], evolve_info["evolve_from"]) + elif "event" in test_location_text: + #logger.info(f"Event: {location['location']}") + self.save_event_encounter(pfic, encounter) + else: + remaining, details = self.extract_additional_information(location["tag"]) + routes, remaining = self.extract_routes(remaining) + #logger.info(f"Routes: {routes}") + #logger.info(f"Remaining: {remaining.strip()}") + #logger.info(f"Details: {details}") + + if len(details["times"]) > 0: + #logger.info("Stupid Data") + pass + + for route in routes: + route_name = f"Route {route}" + self.save_encounter(pfic, encounter, route_name, details["days"], details["times"], details["dual_slot"], details["static_encounter"], details["static_encounter_count"], details["extra_text"], details["stars"], details["Rods"], details["Fishing"], details["starter"] ) + + if remaining != "": + remaining_locations = remaining.replace(" and ", ",").split(",") + for remaining_location in remaining_locations: + if remaining_location.strip() == "": + continue + + ignore_location = False + for ignore in self.encounters_to_ignore: + if ignore in remaining_location.lower(): + ignore_location = True + break + + if ignore_location: + continue + + self.save_encounter(pfic, encounter, remaining_location.strip(), details["days"], details["times"], details["dual_slot"], details["static_encounter"], details["static_encounter_count"], details["extra_text"], details["stars"], details["Rods"], details["Fishing"], details["starter"] ) + + return self.encounters + + + def get_locations_from_bulbapedia(self, pokemon_name, form, force_refresh = False): + url = f"https://bulbapedia.bulbagarden.net/wiki/{pokemon_name}_(Pokémon)" + page_data = cache.fetch_url(url) + if not page_data: + return None + + cache_key = f'locations_{url}_data' + + if force_refresh: + cache.purge(cache_key) + + cached_entry = cache.get(cache_key) + if cached_entry != None: + return cached_entry + + soup = BeautifulSoup(page_data, 'html.parser') + if not soup: + return None + + # Try different methods to find the locations table + locations_table = None + possible_headers = ['Game locations', 'In side games', 'In spin-off games'] + + for header in possible_headers: + span = soup.find('span', id=header.replace(' ', '_')) + if span: + locations_table = span.find_next('table', class_='roundy') + if locations_table: + break + + if not locations_table: + print(f"Warning: Couldn't find locations table for {pokemon_name}") + return None + + raw_game_locations = {} + + generation_tbody = locations_table.find('tbody', recursive=False) + generation_rows = generation_tbody.find_all('tr', recursive=False) + for generation_row in generation_rows: + random_nested_td = generation_row.find('td', recursive=False) + if not random_nested_td: + continue + random_nested_table = random_nested_td.find('table', recursive=False) + if not random_nested_table: + continue + random_nested_tbody = random_nested_table.find('tbody', recursive=False) + random_nested_rows = random_nested_tbody.find_all('tr', recursive=False) + + for nested_row in random_nested_rows: + if 'Generation' in nested_row.get_text(strip=True): + continue + + games_container_td = nested_row.find('td', recursive=False) + if not games_container_td: + continue + games_container_table = games_container_td.find('table', recursive=False) + if not games_container_table: + continue + games_container_tbody = games_container_table.find('tbody', recursive=False) + games_container_rows = games_container_tbody.find_all('tr', recursive=False) + for games_container_row in games_container_rows: + games = games_container_row.find_all('th') + for game in games: + raw_game = game.get_text(strip=True) + if is_mainline_game(raw_game) == None: + continue + locations_container_td = games_container_row.find('td', recursive=False) + if not locations_container_td: + continue + locations_container_table = locations_container_td.find('table', recursive=False) + if not locations_container_table: + continue + locations_container_tbody = locations_container_table.find('tbody', recursive=False) + locations = locations_container_tbody.find_all('td') + for location in locations: + groups = self.split_td_contents(location) + for group in groups: + if raw_game not in raw_game_locations: + raw_game_locations[raw_game] = [] + raw_game_locations[raw_game].append(group) + + # Process events + events_section = soup.find('span', id='In_events') + event_tables = self.process_event_tables(events_section) if events_section else {} + + game_locations = {} + for raw_game, raw_locations in raw_game_locations.items(): + encounters = self.process_game_locations(raw_game, raw_locations, form) + if encounters and len(encounters) > 0: + game_locations[raw_game] = encounters + + # Process event tables + for variant in event_tables: + if (variant == pokemon_name and form is None) or (form and form in variant): + self.process_event_table(event_tables[variant], game_locations) + + cache.set(cache_key, game_locations) + return game_locations + + def split_td_contents(self, td): + groups = [] + current_group = [] + for content in td.contents: + if isinstance(content, NavigableString): + text = content.strip() + if text: + current_group.append(content) + elif content.name == 'br': + if current_group: + groups.append(''.join(str(item) for item in current_group)) + current_group = [] + else: + current_group.append(content) + if current_group: + groups.append(''.join(str(item) for item in current_group)) + return groups + + def process_game_locations(self, raw_game, raw_locations, form): + locations = [] + + for raw_location in raw_locations: + raw_text = raw_location + forms = self.parse_form_information(raw_location) + if form is None: + if len(forms) > 0: + for form_info in forms: + main_form = form_info["main_form"] + if default_forms and main_form and main_form in self.default_forms_set: + main_form = None + + if main_form and (main_form != "All Forms" and main_form != "Kantonian Form" and main_form != "All Sizes"): + continue + + locations.append({"location": raw_text, "tag": raw_location}) + else: + locations.append({"location": raw_text, "tag": raw_location}) + elif len(forms) > 0: + for form_info in forms: + if self.form_matches(form_info, form, default_forms): + locations.append({"location": raw_text, "tag": raw_location}) + else: + form_info = {"main_form": None, "sub_form": None, "region": None} + if self.form_matches(form_info, form, default_forms): + locations.append({"location": raw_text, "tag": raw_location}) + + return locations if locations else None + + def process_event_tables(self, events_section): + event_tables = {} + if events_section: + next_element = events_section.parent.find_next_sibling() + while next_element and next_element.name != 'h3': + if next_element.name == 'h5': + variant = next_element.text.strip() + table = next_element.find_next_sibling('table', class_='roundy') + if table: + event_tables[variant] = table + next_element = next_element.find_next_sibling() + return event_tables + + def parse_form_information(self, html_content): + soup = BeautifulSoup(html_content, 'html.parser') + + #TODO: This wont work for lines that have several small blocks in one line. + #TODO: Adjust this to handle more than one small block, see Basculin for example + small_tag = soup.find('small') + + forms = [] + # Form info is in bold inside a small tag. + if small_tag: + bold_tags = small_tag.find_all('b') + for bold_tag in bold_tags: + form_text = bold_tag.get_text(strip=True) + + # Remove parentheses + form_text = form_text.strip('()') + + if "/" in form_text: + last_word = singularize(form_text.split()[-1]) + form_text = form_text.replace(last_word, "").strip() + parts = form_text.split('/') + for part in parts: + main_form = part.strip() + " " + last_word + info = { + "main_form": main_form, + "sub_form": None + } + forms.append(info) + continue + + # Split the text into main form and breed (if present) + parts = form_text.split('(') + main_form = parts[0].strip() + + # "Factor"s are not actual forms, they are properties of the pokemon you can encoutner. + if main_form and "factor" in main_form.lower(): + continue + + breed = parts[1].strip(')') if len(parts) > 1 else None + + info = { + "main_form": main_form, + "sub_form": breed + } + + for region in regional_descriptors: + if region in main_form.lower(): + info["region"] = region + break + + forms.append(info) + else: #..... Gimmighoul + headings = soup.find_all('b') + if len(headings) > 0: + for heading in headings: + if heading.parent.name == 'sup': + continue + if "form" not in heading.get_text(strip=True).lower(): + continue + main_form = heading.get_text(strip=True) + info = { + "main_form": main_form, + "sub_form": None + } + + for region in regional_descriptors: + if region in main_form.lower(): + info["region"] = region + break + + forms.append(info) + + return forms + + def form_matches(self, form_info, form, default_forms): + main_form = form_info["main_form"] + sub_form = form_info["sub_form"] + try: + region = form_info['region'] if 'region' in form_info else None + except KeyError: + region = None + + if default_forms and main_form and main_form in default_forms: + main_form = None + + if form.lower() in ["spring form", "summer form", "autumn form", "winter form"] and main_form == None: + return True + + if main_form is None: + return False + + if main_form in ["All Forms", "All Sizes"]: + return True + + if region == None and main_form in ["Kantonian Form"]: + return True + + main_form_match = compare_pokemon_forms(form, main_form) or fuzz.partial_ratio(form.lower(), main_form.lower()) >= 95 + sub_form_match = compare_pokemon_forms(form, sub_form) or (sub_form and fuzz.partial_ratio(form.lower(), sub_form.lower()) >= 95) + + if not main_form_match and not sub_form_match and region: + region_match = compare_pokemon_forms(form, region) or fuzz.partial_ratio(form.lower(), region.lower()) >= 95 + return region_match + + return main_form_match or sub_form_match + + def extract_routes(self, s): + # Find all route numbers, including those after "and" or separated by commas + route_pattern = r'Routes?\s+((?:\d+(?:,?\s+(?:and\s+)?)?)+)' + route_match = re.search(route_pattern, s, re.IGNORECASE) + + if route_match: + # Extract all numbers from the matched group + numbers = re.findall(r'\d+', route_match.group(1)) + + # Remove the extracted part from the original string + remaining = s[:route_match.start()] + s[route_match.end():].lstrip(', ') + + return numbers, remaining + else: + return [], s + + def extract_additional_information(self, s): + details = {} + details["days"] = [] + details["times"] = [] + details["dual_slot"] = None + details["static_encounter_count"] = 0 + details["static_encounter"] = False + details["starter"] = False + details["extra_text"] = [] + details["stars"] = [] + details["Fishing"] = False + details["Rods"] = [] + + if s is None: + return "", details + + soup = BeautifulSoup(s, 'html.parser') + full_text = soup.get_text() + sup_tags = soup.find_all('sup') + sup_text = [] + + if "first partner" in full_text.lower(): + details["starter"] = True + + for sup_tag in sup_tags: + text = sup_tag.get_text(strip=True) + + if find_match_in_string_array(text, days): + details["days"].append(text) + sup_text.append(text) + + if find_match_in_string_array(text, times): + details["times"].append(text) + sup_text.append(text) + + bracket_text = extract_bracketed_text(full_text) + + for text in bracket_text: + text = text.strip() + text_lower = text.lower() + + game = is_mainline_game(text_lower) + if game != None: + details["dual_slot"] = game["Name"] + text = re.sub(game["Name"], '', text_lower, flags=re.IGNORECASE) + + match = find_match_in_string_array(text_lower, days) + if match: + details["days"].append(match) + text = re.sub(match, '', text_lower, flags=re.IGNORECASE) + + match = find_match_in_string_array(text_lower, times) + if match: + details["times"].append(match) + text = re.sub(match, '', text_lower, flags=re.IGNORECASE) + + if "only one" in text_lower: + details["static_encounter_count"] = 1 + details["static_encounter"] = True + text = re.sub(r'only one', '', text_lower, flags=re.IGNORECASE) + elif "only two" in text_lower: + details["static_encounter_count"] = 2 + details["static_encounter"] = True + text = re.sub(r'only two', '', text_lower, flags=re.IGNORECASE) + + if "rod" in text_lower: + match = find_match_in_string_array(text_lower, rods) + if match: + details["Fishing"] = True + details["Rods"].append(match) + text = re.sub(match, '', text_lower, flags=re.IGNORECASE) + + if "★" in text: + star_parts = re.findall(r'\d★,*', text) + for part in star_parts: + details["stars"].append(part.replace(',', '').strip()) + text = re.sub(r'\d★,*', '', text) + + if text.strip() != "": + details["extra_text"].append(text.strip()) + sup_text.append(text.strip()) + + if len(sup_text) > 0: + for text in sup_text: + full_text = full_text.replace(text, "") + + if len(bracket_text) > 0: + for text in bracket_text: + full_text = full_text.replace(text, "") + full_text = full_text.replace('(', "").replace(')', "") + + return full_text.strip(), details + else: + return full_text, details + + def extract_evolve_information(self, s: str): + details = {} + if s is None or s == "": + return details + + s = s.replace("Evolve", "") + + parts = s.split(" ") + + if len(parts) >= 1: + target_pokemon = parts[0].strip() + + form = None + if "♀" in target_pokemon: + target_pokemon = target_pokemon.replace("♀", "").strip() + form = "Female" + + if "♂" in target_pokemon: + target_pokemon = target_pokemon.replace("♂", "").strip() + form = "Male" + + results = db.get_pokemon_details_by_name(target_pokemon) + + if results: + for result in results: + if compare_pokemon_forms(result["form_name"], form): + details["evolve_from"] = result["pfic"] + + return details + + def save_evolve_encounter(self, pfic, game, days, times, from_pfic): + game_id = db.get_game_id_by_name(game) + + encounter = { + "pfic": pfic, + "game_id": game_id, + "type": "evolve", + "data": { + "day": None, + "time": None, + "from_pfic": from_pfic, + } + } + + if len(days) > 0: + for day in days: + encounter["data"]["day"] = day + encounter["data"]["time"] = None + self.encounters.append(encounter) + + elif len(times) > 0: + for time in times: + encounter["data"]["day"] = None + encounter["data"]["time"] = time + self.encounters.append(encounter) + else: + encounter["data"]["day"] = None + encounter["data"]["time"] = None + self.encounters.append(encounter) + + def save_event_encounter(self, pfic, game): + game_id = db.get_game_id_by_name(game) + + encounter = { + "pfic": pfic, + "game_id": game_id, + "type": "event" + } + + self.encounters.append(encounter) + + def save_encounter(self, pfic, game, location, days, times, dual_slot, static_encounter, static_encounter_count, extra_text, stars, rods, fishing, starter): + game_id = db.get_game_id_by_name(game) + extra_text_str = ' '.join(extra_text) if extra_text else None + stars_str = ','.join(sorted(stars)) if stars else None + rods_str = ','.join(sorted(rods)) if rods else None + + encounter_type = "random" + + if starter: + encounter_type = "starter" + + if static_encounter: + encounter_type = "static" + + encounter = { + "pfic": pfic, + "game_id": game_id, + "type": encounter_type, + "data": { + "location": location, + "day": None, + "time": None, + "dual_slot": dual_slot, + "extra_text": extra_text_str, + "stars": stars_str, + "rods": rods_str, + "fishing": fishing + } + } + + if static_encounter: + encounter["data"]["static_encounter_count"] = static_encounter_count + + if len(days) > 0: + for day in days: + encounter["data"]["day"] = day + encounter["data"]["time"] = None + self.encounters.append(encounter) + + elif len(times) > 0: + for time in times: + encounter["data"]["day"] = None + encounter["data"]["time"] = time + self.encounters.append(encounter) + + else: + encounter["data"]["day"] = None + encounter["data"]["time"] = None + self.encounters.append(encounter) + + def process_event_tables(self, events_section): + event_tables = {} + if events_section: + next_element = events_section.parent.find_next_sibling() + while next_element and next_element.name != 'h3': + if next_element.name == 'h5': + variant = next_element.text.strip() + table = next_element.find_next_sibling('table', class_='roundy') + if table: + event_tables[variant] = table + next_element = next_element.find_next_sibling() + return event_tables + + def process_event_table(self, table, game_locations): + for row in table.find_all('tr')[1:]: # Skip header row + cells = row.find_all('td') + if len(cells) >= 6: # Ensure all required columns are present + # Extract game names as a list + game_links = cells[0].find_all('a') + individual_games = [] + + for link in game_links: + # Replace specific known prefixes + game_name = link['title'].replace("Pokémon ", "").replace("Versions", "").replace(" Version", "").replace(" (Japanese)", "") + + # Split on " and ", which is used for combined games + parsed_names = game_name.split(" and ") + + # Add the parsed names to the list + individual_games.extend(parsed_names) + + # Print extracted game names for debugging + print(f"Extracted game names from row: {individual_games}") + + # Filter games to include only those in all_games + matching_games = [] + + for game in individual_games: + match = is_mainline_game(game) + if match: + matching_games.append(game) + + # Print matching games for debugging + print(f"Matching games after filtering: {matching_games}") + + if matching_games: + location = cells[2].text.strip() + distribution_period = cells[5].text.strip() + for game in matching_games: + if game not in game_locations: + game_locations[game] = [] + game_locations[game].append({ + "location": f"Event: {location}", + "tag": str(cells[2]) + }) \ No newline at end of file diff --git a/utility/data.py b/utility/data.py index c289958..186c386 100644 --- a/utility/data.py +++ b/utility/data.py @@ -318,4 +318,8 @@ POKEMON_PROPER_NOUNS = { } POKEMON_PROPER_NOUNS = POKEMON_PROPER_NOUNS | set(regions) -POKEMON_PROPER_NOUNS = POKEMON_PROPER_NOUNS | set(regional_descriptors) \ No newline at end of file +POKEMON_PROPER_NOUNS = POKEMON_PROPER_NOUNS | set(regional_descriptors) + +days = ["Mo", "Tu", "We", "Th", "Fr", "Sa", "Su"] +times = ["Morning", "Day", "Night"] +rods = ["Old Rod", "Good Rod", "Super Rod"] \ No newline at end of file diff --git a/utility/functions.py b/utility/functions.py index b54d010..32f3c8b 100644 --- a/utility/functions.py +++ b/utility/functions.py @@ -52,6 +52,16 @@ def find_game_generation(game_name: str) -> int: return game["Generation"] return None +def is_mainline_game(game_name: str): + game_name = game_name.lower() + for game in main_line_games: + if game_name == game["Name"].lower() or game_name in (name.lower() for name in game["AltNames"]): + return game + return None + +def find_match_in_string_array(search_string, string_array): + return next((item for item in string_array if item.lower() == search_string.lower()), None) + def sanitize_filename(filename): # Define a dictionary of symbol replacements symbol_replacements = { @@ -91,4 +101,32 @@ def get_form_name(pokemon, strip_gender = False): form = form.replace("Female", "").replace("Male", "").strip() if form != "": return form - return None \ No newline at end of file + return None + +def extract_bracketed_text(string): + results = [] + stack = [] + start_index = -1 + + for i, char in enumerate(string): + if char == '(': + if not stack: + start_index = i + stack.append(i) + elif char == ')': + if stack: + stack.pop() + if not stack: + results.append(string[start_index + 1:i]) + start_index = -1 + else: + #logger.warning(f"Warning: Unmatched closing parenthesis at position {i}") + pass + + # Handle any remaining unclosed brackets + if stack: + #logger.warning(f"Warning: {len(stack)} unmatched opening parentheses") + for unmatched_start in stack: + results.append(string[unmatched_start + 1:]) + + return results \ No newline at end of file -- 2.30.2 From 0f584e5b1e2d8d47501fff0779a827759ef87cba Mon Sep 17 00:00:00 2001 From: Quildra Date: Sun, 10 Nov 2024 21:21:10 +0000 Subject: [PATCH 16/21] WIP - Moving on to calculating mark information. --- database/db_controller.py | 22 ++++- ui/main_window_controller.py | 8 +- ui/workers/calculate_origin_mark_worker.py | 101 +++++++++++++++++++++ utility/data.py | 14 ++- utility/functions.py | 11 ++- 5 files changed, 151 insertions(+), 5 deletions(-) create mode 100644 ui/workers/calculate_origin_mark_worker.py diff --git a/database/db_controller.py b/database/db_controller.py index 4542c95..3b42f20 100644 --- a/database/db_controller.py +++ b/database/db_controller.py @@ -437,6 +437,16 @@ class DBController: results = self.cursor.fetchall() return [dict(row) for row in results] + def get_game_by_id(self, id): + self.cursor.execute(''' + SELECT * FROM games + WHERE id = ? + ''', (id)) + + # Fetch and print the results + result = self.cursor.fetchone() + return dict(result) + def get_games_by_generation(self, generation): self.cursor.execute(''' SELECT id, name FROM games @@ -462,4 +472,14 @@ class DBController: ''', (pfic, game_id, type, json.dumps(data))) self.conn.commit() print(f"Added: {pfic}") - pass \ No newline at end of file + pass + + def get_encounters(self, pfic): + self.cursor.execute(''' + SELECT * FROM encounters + WHERE PFIC = ? + ''', (pfic)) + + # Fetch and print the results + results = self.cursor.fetchall() + return [dict(row) for row in results] \ No newline at end of file diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index 379510a..f2a20af 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -3,6 +3,7 @@ from PyQt6.QtWidgets import QMenu from PyQt6.QtGui import QAction import os +from ui.workers.calculate_origin_mark_worker import CalculateOriginMarkWorker from ui.workers.gather_encounter_locations import GatherEncountersWorker from ui.workers.gather_home_storage_status_worker import GatherHomeStorageStatus from ui.workers.gather_pokemon_forms_worker import GatherPokemonFormsWorker @@ -153,7 +154,12 @@ class MainWindowController: db.update_encounter_locations(data) def gather_marks_info(self): - pass + worker = CalculateOriginMarkWorker() + worker.signals.finished.connect(self.on_marks_calculated) + self.thread_pool.start(worker) + + def on_marks_calculated(self, data): + print("Works Done!") def load_shiftable_forms(self): pass diff --git a/ui/workers/calculate_origin_mark_worker.py b/ui/workers/calculate_origin_mark_worker.py new file mode 100644 index 0000000..4cdaf40 --- /dev/null +++ b/ui/workers/calculate_origin_mark_worker.py @@ -0,0 +1,101 @@ +from PyQt6.QtCore import QObject, pyqtSignal, QRunnable + +from cache import cache +from db import db + +from utility.functions import get_display_name, get_shiftable_forms + +class CalculateOriginMarkWorkerSignals(QObject): + finished = pyqtSignal(list) + +class CalculateOriginMarkWorker(QRunnable): + def __init__(self): + super().__init__() + self.signals = CalculateOriginMarkWorkerSignals() + self.marks = {} + + def run(self): + try: + gathered_data = self.calculate_marks() + self.signals.finished.emit(gathered_data) + except Exception as e: + print(f"Error gathering Pokémon home storage status: {e}") + + def calculate_marks(self): + all_pokemon_forms = db.get_list_of_pokemon_forms() + for form_entry in all_pokemon_forms: + if form_entry["storable_in_home"] == False: + continue + + print(f"Determining mark for {get_display_name(form_entry)}") + + target_generation = form_entry["generation"] + pfic = form_entry["pfic"] + + #Rule 1 + # 1. If a pokemon form has a previous evolution from within the same generation, + # use the mark of the previous evolution. This should be recursive within the same generation. + chain = db.get_full_evolution_paths(pfic) + if chain: + base_form_in_generation = None + last_pfic = pfic + current_pfic = pfic + while True: + current_pfic, _ = db.get_previous_evolution(current_pfic) + if current_pfic == None: + base_form_in_generation = last_pfic + break + chain_pokemon_data = db.get_pokemon_details(current_pfic) + #chain_pokemon_data = event_system.call_sync('get_pokemon_form_by_pfic', current_pfic[0]) + if chain_pokemon_data["generation"] == target_generation: + base_form_in_generation = current_pfic + else: + base_form_in_generation = last_pfic + break + last_pfic = current_pfic + + if base_form_in_generation and base_form_in_generation != pfic: + print(f"Base form in generation for {get_display_name(form_entry)} is {base_form_in_generation}") + mark_id = self.determine_origin_mark(base_form_in_generation, target_generation) + if mark_id != None: + #event_system.emit_sync('assign_mark_to_form', (pfic, mark_id)) + self.marks[pfic] = mark_id + continue + elif base_form_in_generation == pfic: + mark_id = self.determine_origin_mark(pfic, target_generation) + if mark_id != None: + #event_system.emit_sync('assign_mark_to_form', (pfic, mark_id)) + self.marks[pfic] = mark_id + continue; + + pass + + def determine_origin_mark(self, pfic, target_generation): + shiftable_forms = get_shiftable_forms(pfic) + if len(shiftable_forms) > 0: + for shiftable_form in shiftable_forms: + mark_id = self.determine_origin_mark(shiftable_form[2], target_generation) + return mark_id + encounters = db.get_encounters(pfic) + if encounters: + generation_encounters = [] + for encounter in encounters: + game_info = db.get_game_by_id(encounter["game_id"]) + game_generation = game_info["generation"] + game_id = game_info["id"] + encounter = encounter + (game_generation, game_id) + if game_generation == target_generation: + generation_encounters.append(encounter) + if len(generation_encounters) > 0: + generation_encounters = sorted(generation_encounters, key=lambda x: x[12]) + form_info = db.get_pokemon_details(pfic) + game_info = db.get_game_by_id(encounter["game_id"]) + mark_id = 1 # TODO: Work this bit out. + mark_id = event_system.call_sync('get_mark_for_game_name', generation_encounters[0][0]) + if mark_id == None: + self.logger.info(f"No mark found for {form_info[0]} {form_info[1]}") + else: + mark_details = event_system.call_sync('get_mark_details', mark_id) + self.logger.info(f"Mark for {form_info[0]} {form_info[1]} is {mark_details[0]} - {mark_details[1]}") + return mark_id + return None diff --git a/utility/data.py b/utility/data.py index 186c386..ed88ba4 100644 --- a/utility/data.py +++ b/utility/data.py @@ -322,4 +322,16 @@ POKEMON_PROPER_NOUNS = POKEMON_PROPER_NOUNS | set(regional_descriptors) days = ["Mo", "Tu", "We", "Th", "Fr", "Sa", "Su"] times = ["Morning", "Day", "Night"] -rods = ["Old Rod", "Good Rod", "Super Rod"] \ No newline at end of file +rods = ["Old Rod", "Good Rod", "Super Rod"] + +shiftable_forms = [ + {"from_pfic":"0412-04-002-0", "to_pfic":"0412-04-001-0"}, + {"from_pfic":"0412-04-003-0", "to_pfic":"0412-04-001-0"}, + {"from_pfic":"0641-05-002-0", "to_pfic":"0641-05-001-0"}, + {"from_pfic":"0642-05-002-0", "to_pfic":"0642-05-001-0"}, + {"from_pfic":"0645-05-002-0", "to_pfic":"0645-05-001-0"}, + {"from_pfic":"0647-05-002-0", "to_pfic":"0647-05-001-0"}, + {"from_pfic":"0720-06-002-0", "to_pfic":"0720-06-001-0"}, + {"from_pfic":"0905-08-002-0", "to_pfic":"0905-08-001-0"}, + {"from_pfic":"0492-04-002-0", "to_pfic":"0492-04-001-0"} +] \ No newline at end of file diff --git a/utility/functions.py b/utility/functions.py index 32f3c8b..da2c89a 100644 --- a/utility/functions.py +++ b/utility/functions.py @@ -1,4 +1,4 @@ -from .data import pokemon_generations, main_line_games +from .data import pokemon_generations, main_line_games, shiftable_forms import unicodedata import re @@ -129,4 +129,11 @@ def extract_bracketed_text(string): for unmatched_start in stack: results.append(string[unmatched_start + 1:]) - return results \ No newline at end of file + return results + +def get_shiftable_forms(pfic): + forms = [] + for form_pair in shiftable_forms: + if pfic == form_pair["from_pfic"]: + forms.append(form_pair) + return forms -- 2.30.2 From a130bf6c40ff38687d9df497021a544ac659a78f Mon Sep 17 00:00:00 2001 From: Dan Date: Tue, 12 Nov 2024 15:10:39 +0000 Subject: [PATCH 17/21] - Added mark informations --- database/db_controller.py | 20 ++-- ui/workers/calculate_origin_mark_worker.py | 72 +++++++++---- ui/workers/gather_encounter_locations.py | 12 ++- ui/workers/gather_evolutions_worker.py | 45 +++++---- utility/data.py | 111 ++++++++++++++------- 5 files changed, 176 insertions(+), 84 deletions(-) diff --git a/database/db_controller.py b/database/db_controller.py index 3b42f20..e2c0e0c 100644 --- a/database/db_controller.py +++ b/database/db_controller.py @@ -87,6 +87,7 @@ class DBController: name TEXT NOT NULL UNIQUE, alt_names TEXT, generation INTEGER NOT NULL, + mark TEXT NOT NULL, data JSON ) ''') @@ -95,11 +96,12 @@ class DBController: name = game["Name"] alt_names = ", ".join(game["AltNames"]) # Convert list to comma-separated string generation = game["Generation"] + mark = game["Mark"] cursor.execute(''' - INSERT OR IGNORE INTO games (name, alt_names, generation) - VALUES (?, ?, ?) - ''', (name, alt_names, generation)) + INSERT OR IGNORE INTO games (name, alt_names, generation, mark) + VALUES (?, ?, ?, ?) + ''', (name, alt_names, generation, mark)) def add_pokemon_form(self, pfic, name, form_name, national_dex, generation, sprite_url, gender_relevant): data = { @@ -441,7 +443,7 @@ class DBController: self.cursor.execute(''' SELECT * FROM games WHERE id = ? - ''', (id)) + ''', (id,)) # Fetch and print the results result = self.cursor.fetchone() @@ -474,11 +476,15 @@ class DBController: print(f"Added: {pfic}") pass - def get_encounters(self, pfic): - self.cursor.execute(''' + def get_encounters(self, pfic, type=None): + query = ''' SELECT * FROM encounters WHERE PFIC = ? - ''', (pfic)) + ''' + if type: + query += f"AND type = '{type}'" + + self.cursor.execute(query, (pfic,)) # Fetch and print the results results = self.cursor.fetchall() diff --git a/ui/workers/calculate_origin_mark_worker.py b/ui/workers/calculate_origin_mark_worker.py index 4cdaf40..68b1011 100644 --- a/ui/workers/calculate_origin_mark_worker.py +++ b/ui/workers/calculate_origin_mark_worker.py @@ -1,4 +1,5 @@ from PyQt6.QtCore import QObject, pyqtSignal, QRunnable +import json from cache import cache from db import db @@ -35,8 +36,9 @@ class CalculateOriginMarkWorker(QRunnable): #Rule 1 # 1. If a pokemon form has a previous evolution from within the same generation, # use the mark of the previous evolution. This should be recursive within the same generation. + print("Checking Rule 1") chain = db.get_full_evolution_paths(pfic) - if chain: + if chain and (len(chain["predecessors"]) > 0 or len(chain["successors"]) > 0): base_form_in_generation = None last_pfic = pfic current_pfic = pfic @@ -46,7 +48,6 @@ class CalculateOriginMarkWorker(QRunnable): base_form_in_generation = last_pfic break chain_pokemon_data = db.get_pokemon_details(current_pfic) - #chain_pokemon_data = event_system.call_sync('get_pokemon_form_by_pfic', current_pfic[0]) if chain_pokemon_data["generation"] == target_generation: base_form_in_generation = current_pfic else: @@ -58,44 +59,81 @@ class CalculateOriginMarkWorker(QRunnable): print(f"Base form in generation for {get_display_name(form_entry)} is {base_form_in_generation}") mark_id = self.determine_origin_mark(base_form_in_generation, target_generation) if mark_id != None: - #event_system.emit_sync('assign_mark_to_form', (pfic, mark_id)) self.marks[pfic] = mark_id continue elif base_form_in_generation == pfic: mark_id = self.determine_origin_mark(pfic, target_generation) if mark_id != None: - #event_system.emit_sync('assign_mark_to_form', (pfic, mark_id)) self.marks[pfic] = mark_id - continue; + continue + + #Rule 2 + # If a pokemon form has no previous evolution from within the same generation, + # look at the encounters of the pokemon form from this generation and use the mark of the earliest + # game you can encounter that form in from that generation + print("Checking Rule 2") + mark_id = self.determine_origin_mark(pfic, target_generation) + if mark_id != None: + self.marks[pfic] = mark_id + continue + #Rule 3 + # If there are no encounters for the pokemon form from this generation, + # look to see if a previous evolution has an encounter from this generation, and use the mark of the earliest + # game from this generation that the previous evolution is encounterable in. + print("Checking Rule 3") + mark_id = self.test_evolve_encounters(pfic, target_generation) + if mark_id != None: + self.marks[pfic] = mark_id + continue pass def determine_origin_mark(self, pfic, target_generation): shiftable_forms = get_shiftable_forms(pfic) if len(shiftable_forms) > 0: for shiftable_form in shiftable_forms: - mark_id = self.determine_origin_mark(shiftable_form[2], target_generation) + mark_id = self.determine_origin_mark(shiftable_form["to_pfic"], target_generation) return mark_id encounters = db.get_encounters(pfic) if encounters: generation_encounters = [] for encounter in encounters: game_info = db.get_game_by_id(encounter["game_id"]) - game_generation = game_info["generation"] - game_id = game_info["id"] - encounter = encounter + (game_generation, game_id) - if game_generation == target_generation: + encounter["game"] = game_info + if encounter["game"]["generation"] == target_generation: generation_encounters.append(encounter) if len(generation_encounters) > 0: - generation_encounters = sorted(generation_encounters, key=lambda x: x[12]) + generation_encounters = sorted(generation_encounters, key=lambda x: x["game"]["generation"]) form_info = db.get_pokemon_details(pfic) - game_info = db.get_game_by_id(encounter["game_id"]) - mark_id = 1 # TODO: Work this bit out. - mark_id = event_system.call_sync('get_mark_for_game_name', generation_encounters[0][0]) + game_info = generation_encounters[0]["game"] + mark_id = game_info["mark"] if mark_id == None: - self.logger.info(f"No mark found for {form_info[0]} {form_info[1]}") + #self.logger.info(f"No mark found for {form_info[0]} {form_info[1]}") + print(f"No mark found for {get_display_name(form_info)}") else: - mark_details = event_system.call_sync('get_mark_details', mark_id) - self.logger.info(f"Mark for {form_info[0]} {form_info[1]} is {mark_details[0]} - {mark_details[1]}") + #self.logger.info(f"Mark for {form_info[0]} {form_info[1]} is {mark_id}") + print(f"Mark for {get_display_name(form_info)} is {mark_id}") + return mark_id + return None + + def test_evolve_encounters(self, pfic, target_generation): + evolve_encounters = db.get_encounters(pfic, "evolve") + if evolve_encounters: + available_encounters = [] + for encounter in evolve_encounters: + game_info = db.get_game_by_id(encounter["game_id"]) + if game_info["generation"] == target_generation: + available_encounters.append(encounter) + + if len(available_encounters) > 0: + available_encounters = sorted(available_encounters, key=lambda x: x.game_id) + data = json.loads(available_encounters[0]["data"]) + mark_id = self.determine_origin_mark(data["from_pfic"], target_generation) + if mark_id != None: + return mark_id + + mark_id = self.test_evolve_encounters(data["from_pfic"], target_generation) + if mark_id != None: return mark_id + return None diff --git a/ui/workers/gather_encounter_locations.py b/ui/workers/gather_encounter_locations.py index ca2159e..ef6812d 100644 --- a/ui/workers/gather_encounter_locations.py +++ b/ui/workers/gather_encounter_locations.py @@ -9,6 +9,7 @@ from db import db from utility.data import default_forms, regional_descriptors, days, times, rods from utility.functions import is_mainline_game, compare_pokemon_forms, find_match_in_string_array, extract_bracketed_text +from utility.pokemon_word_ninja import PokemonWordNinja class GatherEncountersWorkerSignals(QObject): finished = pyqtSignal(list) @@ -18,6 +19,7 @@ class GatherEncountersWorker(QRunnable): super().__init__() self.signals = GatherEncountersWorkerSignals() self.default_forms_set = set(default_forms) + self.splitter = PokemonWordNinja() self.encounters_to_ignore = [ "trade", "time capsule", @@ -53,6 +55,7 @@ class GatherEncountersWorker(QRunnable): pfic = form_entry["pfic"] print(f'Processing {name}') + self.splitter.add_custom_word(name) if form and name in form: form = form.replace(name, "").strip() @@ -101,7 +104,7 @@ class GatherEncountersWorker(QRunnable): test_location_text = BeautifulSoup(test_location, 'html.parser').get_text().lower() if "evolve" in test_location_text: remaining, details = self.extract_additional_information(location["tag"]) - evolve_info = self.extract_evolve_information(remaining) + evolve_info = self.extract_evolve_information(remaining, form_entry["form_name"]) if evolve_info: #logger.info(f"Evolve Info: {evolve_info}") @@ -521,7 +524,7 @@ class GatherEncountersWorker(QRunnable): else: return full_text, details - def extract_evolve_information(self, s: str): + def extract_evolve_information(self, s: str, search_form): details = {} if s is None or s == "": return details @@ -549,6 +552,11 @@ class GatherEncountersWorker(QRunnable): if compare_pokemon_forms(result["form_name"], form): details["evolve_from"] = result["pfic"] + if results and "evolve_from" not in details: + for result in results: + if compare_pokemon_forms(result["form_name"], search_form if search_form != form else "Male"): + details["evolve_from"] = result["pfic"] + return details def save_evolve_encounter(self, pfic, game, days, times, from_pfic): diff --git a/ui/workers/gather_evolutions_worker.py b/ui/workers/gather_evolutions_worker.py index 9dfbd11..f51ec19 100644 --- a/ui/workers/gather_evolutions_worker.py +++ b/ui/workers/gather_evolutions_worker.py @@ -2,6 +2,7 @@ from typing import Optional from PyQt6.QtCore import QObject, pyqtSignal, QRunnable from bs4 import BeautifulSoup, Tag from fuzzywuzzy import fuzz +from number_parser import parse_ordinal from cache import cache from db import db @@ -140,16 +141,24 @@ class GatherEvolutions(QRunnable): main_row = rows[0] branch_rows = rows[1:] - def create_stage(td, current_stage_number): + def create_stage(td): pokemon_name = self.extract_pokemon_name(td) evolution_form = self.extract_evolution_form(td, pokemon_name) + stage = self.extract_stage_form(td).replace("Evolution", "").replace("evolution", "").strip() + numberical_stage = -1 + if stage == "Unevolved" or stage == "Baby form": + numberical_stage = 0 + elif stage == "Castoff": + numberical_stage = 1 + else: + numberical_stage = parse_ordinal(stage) return { "pokemon": pokemon_name, "form": evolution_form, "requirement": None, "method": None, "evolves_to": [], - "stage": current_stage_number + "stage": numberical_stage } # Parse main evolution chain @@ -157,11 +166,10 @@ class GatherEvolutions(QRunnable): pending_method_form = None root = None current_stage = None - stage_number = 0 for td in main_row.find_all('td', recursive=False): if td.find('table'): - new_stage = create_stage(td, stage_number) + new_stage = create_stage(td) new_stage["method"] = pending_method new_stage["requirement"] = pending_method_form pending_method = None @@ -170,41 +178,36 @@ class GatherEvolutions(QRunnable): if current_stage: current_stage["evolves_to"].append(new_stage) current_stage = new_stage - stage_number += 1 else: pending_method, pending_method_form = self.extract_evolution_method(td) - # reduce by one to account for an accidental increase by the last one in the chain. - stage_number -= 1 - # Parse branching evolutions for row in branch_rows: branch_method = None pending_method_form = None branch_stage = None - branch_stage_number = stage_number for td in row.find_all('td', recursive=False): if td.find('table'): - new_stage = create_stage(td, branch_stage_number) + new_stage = create_stage(td) new_stage["method"] = branch_method new_stage["requirement"] = pending_method_form branch_method = None if branch_stage: branch_stage["evolves_to"].append(new_stage) + else: + # Find which main chain Pokémon this branch evolves from + attached = False + for main_stage in self.find_stages(root): + if self.should_attach_branch(main_stage, new_stage): + main_stage["evolves_to"].append(new_stage) + attached = True + break + + if not attached: + print(f"Warning: Could not find a suitable attachment point for branch {new_stage['pokemon']}") branch_stage = new_stage - - # Find which main chain Pokémon this branch evolves from - attached = False - for main_stage in self.find_stages(root): - if self.should_attach_branch(main_stage, branch_stage): - main_stage["evolves_to"].append(branch_stage) - attached = True - break - - if not attached: - print(f"Warning: Could not find a suitable attachment point for branch {branch_stage['pokemon']}") else: branch_method, pending_method_form = self.extract_evolution_method(td) diff --git a/utility/data.py b/utility/data.py index ed88ba4..db09dd9 100644 --- a/utility/data.py +++ b/utility/data.py @@ -16,223 +16,260 @@ regional_descriptors = ["Kantonian", "Johtonian", "Hoennian", "Sinnohan", "Unova yellow = { "Name": "Yellow", "AltNames": ["Pokemon Yellow", "Pokémon Yellow", "Y"], - "Generation": 1 + "Generation": 1, + "Mark": "Game Boy" } red = { "Name": "Red", "AltNames": ["Pokemon Red", "Pokémon Red", "R"], - "Generation": 1 + "Generation": 1, + "Mark": "Game Boy" } blue = { "Name": "Blue", "AltNames": ["Pokemon Blue", "Pokémon Blue", "B"], - "Generation": 1 + "Generation": 1, + "Mark": "Game Boy" } crystal = { "Name": "Crystal", "AltNames": ["Pokemon Crystal", "Pokémon Crystal", "C"], - "Generation": 2 + "Generation": 2, + "Mark": "Game Boy" } gold = { "Name": "Gold", "AltNames": ["Pokemon Gold", "Pokémon Gold", "G"], - "Generation": 2 + "Generation": 2, + "Mark": "Game Boy" } silver = { "Name": "Silver", "AltNames": ["Pokemon Silver", "Pokémon Silver", "S"], - "Generation": 2 + "Generation": 2, + "Mark": "Game Boy" } emerald = { "Name": "Emerald", "AltNames": ["Pokemon Emerald", "Pokémon Emerald", "E"], - "Generation": 3 + "Generation": 3, + "Mark": "Markless" } fire_red = { "Name": "FireRed", "AltNames": ["Pokemon FireRed", "Pokémon FireRed", "FR"], - "Generation": 3 + "Generation": 3, + "Mark": "Markless" } leaf_green = { "Name": "LeafGreen", "AltNames": ["Pokemon LeafGreen", "Pokémon LeafGreen", "LG"], - "Generation": 3 + "Generation": 3, + "Mark": "Markless" } ruby = { "Name": "Ruby", "AltNames": ["Pokemon Ruby", "Pokémon Ruby", "R"], - "Generation": 3 + "Generation": 3, + "Mark": "Markless" } sapphire = { "Name": "Sapphire", "AltNames": ["Pokemon Sapphire", "Pokémon Sapphire", "S"], - "Generation": 3 + "Generation": 3, + "Mark": "Markless" } platinum = { "Name": "Platinum", "AltNames": ["Pokemon Platinum", "Pokémon Platinum", "Pt"], - "Generation": 4 + "Generation": 4, + "Mark": "Markless" } heart_gold = { "Name": "HeartGold", "AltNames": ["Pokemon HeartGold", "Pokémon HeartGold", "HG"], - "Generation": 4 + "Generation": 4, + "Mark": "Markless" } soul_silver = { "Name": "SoulSilver", "AltNames": ["Pokemon SoulSilver", "Pokémon SoulSilver", "SS"], - "Generation": 4 + "Generation": 4, + "Mark": "Markless" } diamond = { "Name": "Diamond", "AltNames": ["Pokemon Diamond", "Pokémon Diamond", "D"], - "Generation": 4 + "Generation": 4, + "Mark": "Markless" } pearl = { "Name": "Pearl", "AltNames": ["Pokemon Pearl", "Pokémon Pearl", "P"], - "Generation": 4 + "Generation": 4, + "Mark": "Markless" } black = { "Name": "Black", "AltNames": ["Pokemon Black", "Pokémon Black", "B"], - "Generation": 5 + "Generation": 5, + "Mark": "Markless" } white = { "Name": "White", "AltNames": ["Pokemon White", "Pokémon White", "W"], - "Generation": 5 + "Generation": 5, + "Mark": "Markless" } black_2 = { "Name": "Black 2", "AltNames": ["Pokemon Black 2", "Pokémon Black 2", "B2"], - "Generation": 5 + "Generation": 5, + "Mark": "Markless" } white_2 = { "Name": "White 2", "AltNames": ["Pokemon White 2", "Pokémon White 2", "W2"], - "Generation": 5 + "Generation": 5, + "Mark": "Markless" } x = { "Name": "X", "AltNames": ["Pokemon X", "Pokémon X"], - "Generation": 6 + "Generation": 6, + "Mark": "Kalos" } y = { "Name": "Y", "AltNames": ["Pokemon Y", "Pokémon Y"], - "Generation": 6 + "Generation": 6, + "Mark": "Kalos" } omega_ruby = { "Name": "Omega Ruby", "AltNames": ["Pokemon Omega Ruby", "Pokémon Omega Ruby", "OR"], - "Generation": 6 + "Generation": 6, + "Mark": "Kalos" } alpha_sapphire = { "Name": "Alpha Sapphire", "AltNames": ["Pokemon Alpha Sapphire", "Pokémon Alpha Sapphire", "AS"], - "Generation": 6 + "Generation": 6, + "Mark": "Kalos" } sun = { "Name": "Sun", "AltNames": ["Pokemon Sun", "Pokémon Sun"], - "Generation": 7 + "Generation": 7, + "Mark": "Alola" } moon = { "Name": "Moon", "AltNames": ["Pokemon Moon", "Pokémon Moon"], - "Generation": 7 + "Generation": 7, + "Mark": "Alola" } ultra_sun = { "Name": "Ultra Sun", "AltNames": ["Pokemon Ultra Sun", "Pokémon Ultra Sun", "US"], - "Generation": 7 + "Generation": 7, + "Mark": "Alola" } ultra_moon = { "Name": "Ultra Moon", "AltNames": ["Pokemon Ultra Moon", "Pokémon Ultra Moon", "UM"], - "Generation": 7 + "Generation": 7, + "Mark": "Alola" } sword = { "Name": "Sword", "AltNames": ["Pokemon Sword", "Pokémon Sword", "Expansion Pass", "Expansion Pass (Sword)"], - "Generation": 8 + "Generation": 8, + "Mark": "Galar" } shield = { "Name": "Shield", "AltNames": ["Pokemon Shield", "Pokémon Shield", "Expansion Pass", "Expansion Pass (Shield)"], - "Generation": 8 + "Generation": 8, + "Mark": "Galar" } brilliant_diamond = { "Name": "Brilliant Diamond", "AltNames": ["Pokemon Brilliant Diamond", "Pokémon Brilliant Diamond", "BD"], - "Generation": 8 + "Generation": 8, + "Mark": "Sinnoh" } shining_pearl = { "Name": "Shining Pearl", "AltNames": ["Pokemon Shining Pearl", "Pokémon Shining Pearl", "SP"], - "Generation": 8 + "Generation": 8, + "Mark": "Sinnoh" } legends_arceus = { "Name": "Legends: Arceus", "AltNames": ["Pokemon Legends: Arceus", "Pokémon Legends: Arceus", "LA", "Legends Arceus", "Arceus"], - "Generation": 8 + "Generation": 8, + "Mark": "Hisui" } scarlet = { "Name": "Scarlet", "AltNames": ["Pokemon Scarlet", "Pokémon Scarlet", "The Hidden Treasure of Area Zero", "The Hidden Treasure of Area Zero (Scarlet)", "The Teal Mask", "The Teal Mask (Scarlet)"], - "Generation": 9 + "Generation": 9, + "Mark": "Paldea" } violet = { "Name": "Violet", "AltNames": ["Pokemon Violet", "Pokémon Violet", "The Hidden Treasure of Area Zero", "The Hidden Treasure of Area Zero (Violet)", "The Teal Mask", "The Teal Mask (Violet)"], - "Generation": 9 + "Generation": 9, + "Mark": "Paldea" } lets_go_pikachu = { "Name": "Lets Go Pikachu", "AltNames": [], - "Generation": 8 + "Generation": 8, + "Mark": "Let's Go" } lets_go_eevee = { "Name": "Lets Go Eevee", "AltNames": [], - "Generation": 8 + "Generation": 8, + "Mark": "Let's Go" } main_line_games = [ -- 2.30.2 From 1705e0b0441f00346f908623fcaf249b8e332307 Mon Sep 17 00:00:00 2001 From: Quildra Date: Tue, 12 Nov 2024 21:55:06 +0000 Subject: [PATCH 18/21] - Fix the get_game_id_from_nanme function to do full matches first. Fix Word Ninja --- database/db_controller.py | 20 +++- ui/main_window_controller.py | 6 +- ui/main_window_view.py | 64 ++++++++++++- utility/data.py | 175 +++++++++++++++++++++------------- utility/pokemon_word_ninja.py | 9 +- 5 files changed, 200 insertions(+), 74 deletions(-) diff --git a/database/db_controller.py b/database/db_controller.py index e2c0e0c..71835a5 100644 --- a/database/db_controller.py +++ b/database/db_controller.py @@ -418,13 +418,25 @@ class DBController: return [row['PFIC'] for row in results] def get_game_id_by_name(self, name): + # First try: exact match against the `name` column self.cursor.execute(''' SELECT id, name, generation FROM games - WHERE name LIKE ? OR alt_names LIKE ? - ''', (f"%{name}%", f"%{name}%")) - - # Fetch and print the results + WHERE name = ? + ''', (name,)) + + # Fetch the result result = self.cursor.fetchone() + + # If no exact match found, try matching using `LIKE` with both `name` and `alt_names` + if not result: + self.cursor.execute(''' + SELECT id, name, generation FROM games + WHERE name LIKE ? OR alt_names LIKE ? + ''', (f"%{name}%", f"%{name}%")) + + # Fetch the result from the second query + result = self.cursor.fetchone() + print(f"ID: {result[0]}, Name: {result[1]}, Generation: {result[2]}") return dict(result) diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index f2a20af..19be816 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -195,9 +195,13 @@ class MainWindowController: self.view.image_label.setText("Image not found") self.load_evolution_chain(pfic) - #self.load_encounter_locations(pfic) + self.load_encounter_locations(pfic) self.current_pfic = pfic def load_evolution_chain(self, pfic): chain = db.get_full_evolution_paths(pfic) self.view.update_evolution_tree(chain, pfic) + + def load_encounter_locations(self, pfic): + encounters = db.get_encounters(pfic) + self.view.update_encounter_list(encounters, pfic) diff --git a/ui/main_window_view.py b/ui/main_window_view.py index c2968a2..3454aa0 100644 --- a/ui/main_window_view.py +++ b/ui/main_window_view.py @@ -8,6 +8,7 @@ from .main_window_controller import MainWindowController from utility.functions import get_display_name from db import db +import json class PokemonUI(QWidget): def __init__(self, parent=None): @@ -293,4 +294,65 @@ class PokemonUI(QWidget): if selected_pfic in tree_items: current_item = tree_items[selected_pfic] self.evolution_tree.scrollToItem(current_item) - self.evolution_tree.setCurrentItem(current_item) \ No newline at end of file + self.evolution_tree.setCurrentItem(current_item) + + def update_encounter_list(self, encounters, pfic): + self.locations_tree.clear() + game_items = {} + + for encounter in encounters: + pfic = encounter["PFIC"] + game_id = encounter["game_id"] + type = encounter["type"] + + if type == "event": + continue + + if type == "evolve": + continue + + data = json.loads(encounter["data"]) + game = db.get_game_by_id(game_id) + game_name = game["name"] + location = data["location"] + + if game_name not in game_items: + #print(f'finding generation for {game}') + game_item = QTreeWidgetItem([game_name]) + game_items[game_name] = game_item + # Use generation for sorting, default to 0 if not found + game_item.setData(0, Qt.ItemDataRole.UserRole, game["generation"]) + #print(f'generation for {game} is {generation}') + + location_item = QTreeWidgetItem([location]) + details = [] + if "day" in data: + details.append(f"Day: {data["day"]}") + if "time" in data: + details.append(f"Time: {data["time"]}") + if "dual_slot" in data: + details.append(f"Dual Slot: {data["dual_slot"]}") + if "static_encounter" in data: + details.append(f"Static Encounter (Count: {data["static_encounter_count"]})") + if "extra_text" in data: + details.append(f"Extra: {data["extra_text"]}") + if "stars" in data: + details.append(f"Stars: {data["stars"]}") + if "fishing" in data: + details.append(f"Fishing") + if "rods" in data: + details.append(f"Rods: {data["rods"]}") + + location_item.setText(1, ", ".join(details)) + game_items[game_name].addChild(location_item) + + # Sort game items by generation and add them to the tree + sorted_game_items = sorted(game_items.values(), key=lambda x: x.data(0, Qt.ItemDataRole.UserRole)) + self.locations_tree.addTopLevelItems(sorted_game_items) + self.locations_tree.expandAll() + + # Update the cache for this Pokémon + #self.encounter_cache[pfic] = len(encounters) > 0 + + # After updating the locations tree + #self.update_pokemon_list_highlights() \ No newline at end of file diff --git a/utility/data.py b/utility/data.py index db09dd9..df4a40c 100644 --- a/utility/data.py +++ b/utility/data.py @@ -15,261 +15,298 @@ regional_descriptors = ["Kantonian", "Johtonian", "Hoennian", "Sinnohan", "Unova yellow = { "Name": "Yellow", - "AltNames": ["Pokemon Yellow", "Pokémon Yellow", "Y"], + "AltNames": ["Pokemon Yellow", "Pokémon Yellow"], "Generation": 1, - "Mark": "Game Boy" + "Mark": "Game Boy", + "Abvr": "Y" } red = { "Name": "Red", - "AltNames": ["Pokemon Red", "Pokémon Red", "R"], + "AltNames": ["Pokemon Red", "Pokémon Red"], "Generation": 1, - "Mark": "Game Boy" + "Mark": "Game Boy", + "Abvr": "R" } blue = { "Name": "Blue", - "AltNames": ["Pokemon Blue", "Pokémon Blue", "B"], + "AltNames": ["Pokemon Blue", "Pokémon Blue"], "Generation": 1, - "Mark": "Game Boy" + "Mark": "Game Boy", + "Abvr": "B" } crystal = { "Name": "Crystal", - "AltNames": ["Pokemon Crystal", "Pokémon Crystal", "C"], + "AltNames": ["Pokemon Crystal", "Pokémon Crystal"], "Generation": 2, - "Mark": "Game Boy" + "Mark": "Game Boy", + "Abvr": "C" } gold = { "Name": "Gold", - "AltNames": ["Pokemon Gold", "Pokémon Gold", "G"], + "AltNames": ["Pokemon Gold", "Pokémon Gold"], "Generation": 2, - "Mark": "Game Boy" + "Mark": "Game Boy", + "Abvr": "G" } silver = { "Name": "Silver", - "AltNames": ["Pokemon Silver", "Pokémon Silver", "S"], + "AltNames": ["Pokemon Silver", "Pokémon Silver"], "Generation": 2, - "Mark": "Game Boy" + "Mark": "Game Boy", + "Abvr": "S" } emerald = { "Name": "Emerald", - "AltNames": ["Pokemon Emerald", "Pokémon Emerald", "E"], + "AltNames": ["Pokemon Emerald", "Pokémon Emerald"], "Generation": 3, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "E" } fire_red = { "Name": "FireRed", - "AltNames": ["Pokemon FireRed", "Pokémon FireRed", "FR"], + "AltNames": ["Pokemon FireRed", "Pokémon FireRed"], "Generation": 3, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "FR" } leaf_green = { "Name": "LeafGreen", - "AltNames": ["Pokemon LeafGreen", "Pokémon LeafGreen", "LG"], + "AltNames": ["Pokemon LeafGreen", "Pokémon LeafGreen"], "Generation": 3, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "LG" } ruby = { "Name": "Ruby", - "AltNames": ["Pokemon Ruby", "Pokémon Ruby", "R"], + "AltNames": ["Pokemon Ruby", "Pokémon Ruby"], "Generation": 3, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "R" } sapphire = { "Name": "Sapphire", - "AltNames": ["Pokemon Sapphire", "Pokémon Sapphire", "S"], + "AltNames": ["Pokemon Sapphire", "Pokémon Sapphire"], "Generation": 3, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "S" } platinum = { "Name": "Platinum", - "AltNames": ["Pokemon Platinum", "Pokémon Platinum", "Pt"], + "AltNames": ["Pokemon Platinum", "Pokémon Platinum"], "Generation": 4, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "Pt" } heart_gold = { "Name": "HeartGold", - "AltNames": ["Pokemon HeartGold", "Pokémon HeartGold", "HG"], + "AltNames": ["Pokemon HeartGold", "Pokémon HeartGold"], "Generation": 4, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "HG" } soul_silver = { "Name": "SoulSilver", - "AltNames": ["Pokemon SoulSilver", "Pokémon SoulSilver", "SS"], + "AltNames": ["Pokemon SoulSilver", "Pokémon SoulSilver"], "Generation": 4, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "SS" } diamond = { "Name": "Diamond", - "AltNames": ["Pokemon Diamond", "Pokémon Diamond", "D"], + "AltNames": ["Pokemon Diamond", "Pokémon Diamond"], "Generation": 4, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "D" } pearl = { "Name": "Pearl", - "AltNames": ["Pokemon Pearl", "Pokémon Pearl", "P"], + "AltNames": ["Pokemon Pearl", "Pokémon Pearl"], "Generation": 4, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "P" } black = { "Name": "Black", - "AltNames": ["Pokemon Black", "Pokémon Black", "B"], + "AltNames": ["Pokemon Black", "Pokémon Black"], "Generation": 5, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "B" } white = { "Name": "White", - "AltNames": ["Pokemon White", "Pokémon White", "W"], + "AltNames": ["Pokemon White", "Pokémon White"], "Generation": 5, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "W" } black_2 = { "Name": "Black 2", - "AltNames": ["Pokemon Black 2", "Pokémon Black 2", "B2"], + "AltNames": ["Pokemon Black 2", "Pokémon Black 2"], "Generation": 5, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "B2" } white_2 = { "Name": "White 2", - "AltNames": ["Pokemon White 2", "Pokémon White 2", "W2"], + "AltNames": ["Pokemon White 2", "Pokémon White 2"], "Generation": 5, - "Mark": "Markless" + "Mark": "Markless", + "Abvr": "W2" } x = { "Name": "X", "AltNames": ["Pokemon X", "Pokémon X"], "Generation": 6, - "Mark": "Kalos" + "Mark": "Kalos", + "Abvr": "X" } y = { "Name": "Y", "AltNames": ["Pokemon Y", "Pokémon Y"], "Generation": 6, - "Mark": "Kalos" + "Mark": "Kalos", + "Abvr": "Y" } omega_ruby = { "Name": "Omega Ruby", - "AltNames": ["Pokemon Omega Ruby", "Pokémon Omega Ruby", "OR"], + "AltNames": ["Pokemon Omega Ruby", "Pokémon Omega Ruby"], "Generation": 6, - "Mark": "Kalos" + "Mark": "Kalos", + "Abvr": "OR" } alpha_sapphire = { "Name": "Alpha Sapphire", - "AltNames": ["Pokemon Alpha Sapphire", "Pokémon Alpha Sapphire", "AS"], + "AltNames": ["Pokemon Alpha Sapphire", "Pokémon Alpha Sapphire"], "Generation": 6, - "Mark": "Kalos" + "Mark": "Kalos", + "Abvr": "AS" } sun = { "Name": "Sun", "AltNames": ["Pokemon Sun", "Pokémon Sun"], "Generation": 7, - "Mark": "Alola" + "Mark": "Alola", + "Abvr": "" } moon = { "Name": "Moon", "AltNames": ["Pokemon Moon", "Pokémon Moon"], "Generation": 7, - "Mark": "Alola" + "Mark": "Alola", + "Abvr": "" } ultra_sun = { "Name": "Ultra Sun", - "AltNames": ["Pokemon Ultra Sun", "Pokémon Ultra Sun", "US"], + "AltNames": ["Pokemon Ultra Sun", "Pokémon Ultra Sun"], "Generation": 7, - "Mark": "Alola" + "Mark": "Alola", + "Abvr": "US" } ultra_moon = { "Name": "Ultra Moon", - "AltNames": ["Pokemon Ultra Moon", "Pokémon Ultra Moon", "UM"], + "AltNames": ["Pokemon Ultra Moon", "Pokémon Ultra Moon"], "Generation": 7, - "Mark": "Alola" + "Mark": "Alola", + "Abvr": "UM" } sword = { "Name": "Sword", "AltNames": ["Pokemon Sword", "Pokémon Sword", "Expansion Pass", "Expansion Pass (Sword)"], "Generation": 8, - "Mark": "Galar" + "Mark": "Galar", + "Abvr": "" } shield = { "Name": "Shield", "AltNames": ["Pokemon Shield", "Pokémon Shield", "Expansion Pass", "Expansion Pass (Shield)"], "Generation": 8, - "Mark": "Galar" + "Mark": "Galar", + "Abvr": "" } brilliant_diamond = { "Name": "Brilliant Diamond", - "AltNames": ["Pokemon Brilliant Diamond", "Pokémon Brilliant Diamond", "BD"], + "AltNames": ["Pokemon Brilliant Diamond", "Pokémon Brilliant Diamond"], "Generation": 8, - "Mark": "Sinnoh" + "Mark": "Sinnoh", + "Abvr": "BD" } shining_pearl = { "Name": "Shining Pearl", - "AltNames": ["Pokemon Shining Pearl", "Pokémon Shining Pearl", "SP"], + "AltNames": ["Pokemon Shining Pearl", "Pokémon Shining Pearl"], "Generation": 8, - "Mark": "Sinnoh" + "Mark": "Sinnoh", + "Abvr": "SP" } legends_arceus = { "Name": "Legends: Arceus", - "AltNames": ["Pokemon Legends: Arceus", "Pokémon Legends: Arceus", "LA", "Legends Arceus", "Arceus"], + "AltNames": ["Pokemon Legends: Arceus", "Pokémon Legends: Arceus", "Legends Arceus", "Arceus"], "Generation": 8, - "Mark": "Hisui" + "Mark": "Hisui", + "Abvr": "LA" } scarlet = { "Name": "Scarlet", "AltNames": ["Pokemon Scarlet", "Pokémon Scarlet", "The Hidden Treasure of Area Zero", "The Hidden Treasure of Area Zero (Scarlet)", "The Teal Mask", "The Teal Mask (Scarlet)"], "Generation": 9, - "Mark": "Paldea" + "Mark": "Paldea", + "Abvr": "" } violet = { "Name": "Violet", "AltNames": ["Pokemon Violet", "Pokémon Violet", "The Hidden Treasure of Area Zero", "The Hidden Treasure of Area Zero (Violet)", "The Teal Mask", "The Teal Mask (Violet)"], "Generation": 9, - "Mark": "Paldea" + "Mark": "Paldea", + "Abvr": "" } lets_go_pikachu = { "Name": "Lets Go Pikachu", "AltNames": [], "Generation": 8, - "Mark": "Let's Go" + "Mark": "Let's Go", + "Abvr": "" } lets_go_eevee = { "Name": "Lets Go Eevee", "AltNames": [], "Generation": 8, - "Mark": "Let's Go" + "Mark": "Let's Go", + "Abvr": "" } main_line_games = [ @@ -351,7 +388,15 @@ POKEMON_PROPER_NOUNS = { "Augurite", "Electirizer", "Magmarizer", - "Gigantamax" + "Gigantamax", + "Hangry", + "Amped", + "Eternamax", + "Terastal", + "Pa'u", + "Sensu", + "Debutante", + "Douse" } POKEMON_PROPER_NOUNS = POKEMON_PROPER_NOUNS | set(regions) diff --git a/utility/pokemon_word_ninja.py b/utility/pokemon_word_ninja.py index 2ff87f4..55b16df 100644 --- a/utility/pokemon_word_ninja.py +++ b/utility/pokemon_word_ninja.py @@ -31,12 +31,15 @@ class PokemonWordNinja: def split(self, text: str) -> str: working_text = text + working_text = working_text.replace("-", " ") # First handle exact custom words to preserve capitalization for word in self.custom_words: - placeholder = self.word_to_placeholder_map[word] - pattern = re.compile(re.escape(word), re.IGNORECASE) - working_text = pattern.sub(placeholder, working_text) + # Use word boundaries to make sure we only match full words + pattern = re.compile(r'\b' + re.escape(word) + r'\b', re.IGNORECASE) + if pattern.search(working_text): + placeholder = self.word_to_placeholder_map[word] + working_text = pattern.sub(placeholder, working_text) # Clean up spaces working_text = ' '.join(working_text.split()) -- 2.30.2 From 29d2b9e8d11c7ae6caa99d8f832221744da556bf Mon Sep 17 00:00:00 2001 From: Dan Date: Wed, 13 Nov 2024 11:39:00 +0000 Subject: [PATCH 19/21] - Removal needs to happen before comparisions --- ui/workers/gather_encounter_locations.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ui/workers/gather_encounter_locations.py b/ui/workers/gather_encounter_locations.py index ef6812d..d862374 100644 --- a/ui/workers/gather_encounter_locations.py +++ b/ui/workers/gather_encounter_locations.py @@ -60,8 +60,14 @@ class GatherEncountersWorker(QRunnable): if form and name in form: form = form.replace(name, "").strip() + if form and form.startswith("Female"): + form = form.replace("Female", "").strip() + + if form and form.startswith("Male"): + form = form.replace("Male", "").strip() + if form and form in default_forms: - form = None + form = None if name == "Unown" and (form != "!" and form != "?"): form = None @@ -78,12 +84,6 @@ class GatherEncountersWorker(QRunnable): if name.lower() == "ho-oh": name = "Ho-Oh" - if form and form.startswith("Female"): - form = form.replace("Female", "").strip() - - if form and form.startswith("Male"): - form = form.replace("Male", "").strip() - if form == "": form = None -- 2.30.2 From c4231271f968681cc1a01e689a824541f001296a Mon Sep 17 00:00:00 2001 From: Dan Date: Wed, 13 Nov 2024 13:54:04 +0000 Subject: [PATCH 20/21] - Finish of the mark calculation - Fix up Milcerys evolution line as its mental --- ui/workers/calculate_origin_mark_worker.py | 64 +++++++++++++++++++++ ui/workers/gather_evolutions_worker.py | 15 ++++- utility/data.py | 66 ++++++++++++++++++++++ 3 files changed, 144 insertions(+), 1 deletion(-) diff --git a/ui/workers/calculate_origin_mark_worker.py b/ui/workers/calculate_origin_mark_worker.py index 68b1011..e2d70d1 100644 --- a/ui/workers/calculate_origin_mark_worker.py +++ b/ui/workers/calculate_origin_mark_worker.py @@ -86,6 +86,70 @@ class CalculateOriginMarkWorker(QRunnable): if mark_id != None: self.marks[pfic] = mark_id continue + + #Rule 3b + # Check to see if this is a sub-form pokemon, and if so, use the mark of the base form. + random_encounters = db.get_encounters(pfic, "random") + static_encounters = db.get_encounters(pfic, "static") + encounters = [] + encounters.extend(random_encounters) + encounters.extend(static_encounters) + count = 0 + if encounters: + for encounter in encounters: + game_info = db.get_game_by_id(encounter["game_id"]) + if game_info["generation"] == target_generation: + count += 1 + if count == 0: + + shiftable_forms = get_shiftable_forms(pfic) + if len(shiftable_forms) > 0: + form_found = False + for shiftable_form in shiftable_forms: + mark_id = self.determine_origin_mark(shiftable_form["to_pfic"], target_generation) + if mark_id != None: + self.marks[pfic] = mark_id + form_found = True + break + if form_found: + continue + + #Rule 4 + # If there are no encounters for the pokemon form or its evolution line from this generation, + # use the mark of the earliest game of the generation is marked as being introducted in. + if encounters: + earliest_game = 100 + for encounter in encounters: + game_info = db.get_game_by_id(encounter["game_id"]) + if game_info["id"] <= earliest_game: + earliest_game = game_info["id"] + if earliest_game < 100: + form_info = db.get_pokemon_details(pfic) + mark_id = game_info["mark"] + if mark_id == None: + print(f"No mark found for {get_display_name(form_info)}") + pass + else: + print(f"Mark for {get_display_name(form_info)} is {mark_id}") + self.marks[pfic] = mark_id + continue + + event_encounters = db.get_encounters(pfic, "event") + if event_encounters: + earliest_game = 100 + for encounter in event_encounters: + game_info = game_info = db.get_game_by_id(encounter["game_id"]) + if game_info["id"] <= earliest_game: + earliest_game = game_info["id"] + if earliest_game < 100: + form_info = db.get_pokemon_details(pfic) + mark_id = game_info["mark"] + if mark_id == None: + print(f"No mark found for {get_display_name(form_info)}") + else: + print(f"Mark for {get_display_name(form_info)} is {mark_id}") + self.marks[pfic] = mark_id + continue; pass def determine_origin_mark(self, pfic, target_generation): diff --git a/ui/workers/gather_evolutions_worker.py b/ui/workers/gather_evolutions_worker.py index f51ec19..0f1da14 100644 --- a/ui/workers/gather_evolutions_worker.py +++ b/ui/workers/gather_evolutions_worker.py @@ -9,7 +9,7 @@ from db import db import re from utility.functions import get_form_name, get_display_name, parse_pfic -from utility.data import non_evolution_forms +from utility.data import non_evolution_forms, alcremie_forms class GatherEvolutionsWorkerSignals(QObject): finished = pyqtSignal(dict) @@ -94,6 +94,19 @@ class GatherEvolutions(QRunnable): else: evolution_tree = self.parse_evolution_chain(evolution_table, pokemon_form) + if evolution_tree["pokemon"] == "Milcery": + evolution_tree["evolves_to"] = [] + for alcremie_form in alcremie_forms: + node = { + "pokemon": "Alcremie", + "form": alcremie_form, + "requirement": None, + "method": "Complicated", + "evolves_to": [], + "stage": 1 + } + evolution_tree["evolves_to"].append(node) + cacheable_container = {} if evolution_tree: self.traverse_and_store(evolution_tree, cacheable_container, gender) diff --git a/utility/data.py b/utility/data.py index df4a40c..52b4c1c 100644 --- a/utility/data.py +++ b/utility/data.py @@ -416,4 +416,70 @@ shiftable_forms = [ {"from_pfic":"0720-06-002-0", "to_pfic":"0720-06-001-0"}, {"from_pfic":"0905-08-002-0", "to_pfic":"0905-08-001-0"}, {"from_pfic":"0492-04-002-0", "to_pfic":"0492-04-001-0"} +] + +alcremie_forms = [ + "Caramel Swirl Berry Sweet", + "Caramel Swirl Clover Sweet", + "Caramel Swirl Flower Sweet", + "Caramel Swirl Love Sweet", + "Caramel Swirl Ribbon Sweet", + "Caramel Swirl Star Sweet", + "Caramel Swirl Strawberry Sweet", + "Vannila Cream Berry Sweet", + "Vannila Cream Clover Sweet", + "Vannila Cream Flower Sweet", + "Vannila Cream Love Sweet", + "Vannila Cream Ribbon Sweet", + "Vannila Cream Star Sweet", + "Vannila Cream Strawberry Sweet", + "Lemon Cream Berry Sweet", + "Lemon Cream Clover Sweet", + "Lemon Cream Flower Sweet", + "Lemon Cream Love Sweet", + "Lemon Cream Ribbon Sweet", + "Lemon Cream Star Sweet", + "Lemon Cream Strawberry Sweet", + "Matcha Cream Berry Sweet", + "Matcha Cream Clover Sweet", + "Matcha Cream Flower Sweet", + "Matcha Cream Love Sweet", + "Matcha Cream Ribbon Sweet", + "Matcha Cream Star Sweet", + "Matcha Cream Strawberry Sweet", + "Mint Cream Berry Sweet", + "Mint Cream Clover Sweet", + "Mint Cream Flower Sweet", + "Mint Cream Love Sweet", + "Mint Cream Ribbon Sweet", + "Mint Cream Star Sweet", + "Mint Cream Strawberry Sweet", + "Rainbow Swirl Berry Sweet", + "Rainbow Swirl Clover Sweet", + "Rainbow Swirl Flower Sweet", + "Rainbow Swirl Love Sweet", + "Rainbow Swirl Ribbon Sweet", + "Rainbow Swirl Star Sweet", + "Rainbow Swirl Strawberry Sweet", + "Ruby Cream Berry Sweet", + "Ruby Cream Clover Sweet", + "Ruby Cream Flower Sweet", + "Ruby Cream Love Sweet", + "Ruby Cream Ribbon Sweet", + "Ruby Cream Star Sweet", + "Ruby Cream Strawberry Sweet", + "Ruby Swirl Berry Sweet", + "Ruby Swirl Clover Sweet", + "Ruby Swirl Flower Sweet", + "Ruby Swirl Love Sweet", + "Ruby Swirl Ribbon Sweet", + "Ruby Swirl Star Sweet", + "Ruby Swirl Strawberry Sweet", + "Salted Cream Berry Sweet", + "Salted Cream Clover Sweet", + "Salted Cream Flower Sweet", + "Salted Cream Love Sweet", + "Salted Cream Ribbon Sweet", + "Salted Cream Star Sweet", + "Salted Cream Strawberry Sweet", ] \ No newline at end of file -- 2.30.2 From 2436180e637b22c69a8cf5303a487d94c25a027a Mon Sep 17 00:00:00 2001 From: Quildra Date: Wed, 13 Nov 2024 21:58:40 +0000 Subject: [PATCH 21/21] - Marks done, still needs a verfication pass --- database/db_controller.py | 4 ++++ ui/main_window_controller.py | 3 +++ ui/workers/calculate_origin_mark_worker.py | 7 ++++--- ui/workers/gather_evolutions_worker.py | 4 ++-- ui/workers/gather_pokemon_forms_worker.py | 2 +- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/database/db_controller.py b/database/db_controller.py index 71835a5..c888de0 100644 --- a/database/db_controller.py +++ b/database/db_controller.py @@ -195,6 +195,10 @@ class DBController: self.update_pokemon_field(pfic, "storable_in_home", status) pass + def update_mark(self, pfic, data): + self.update_pokemon_field(pfic, "mark", data) + pass + def update_pokemon_field(self, pfic, field_name, new_value): # Fetch the existing record self.cursor.execute('SELECT data FROM pokemon_forms WHERE PFIC = ?', (pfic,)) diff --git a/ui/main_window_controller.py b/ui/main_window_controller.py index 19be816..12be192 100644 --- a/ui/main_window_controller.py +++ b/ui/main_window_controller.py @@ -91,6 +91,7 @@ class MainWindowController: pass def save_changes(self): + db.save_changes() pass def export_database(self): @@ -159,6 +160,8 @@ class MainWindowController: self.thread_pool.start(worker) def on_marks_calculated(self, data): + for key in data: + db.update_mark(key, data[key]) print("Works Done!") def load_shiftable_forms(self): diff --git a/ui/workers/calculate_origin_mark_worker.py b/ui/workers/calculate_origin_mark_worker.py index e2d70d1..87486de 100644 --- a/ui/workers/calculate_origin_mark_worker.py +++ b/ui/workers/calculate_origin_mark_worker.py @@ -7,7 +7,7 @@ from db import db from utility.functions import get_display_name, get_shiftable_forms class CalculateOriginMarkWorkerSignals(QObject): - finished = pyqtSignal(list) + finished = pyqtSignal(dict) class CalculateOriginMarkWorker(QRunnable): def __init__(self): @@ -149,8 +149,9 @@ class CalculateOriginMarkWorker(QRunnable): else: print(f"Mark for {get_display_name(form_info)} is {mark_id}") self.marks[pfic] = mark_id - continue; - pass + continue + + return self.marks def determine_origin_mark(self, pfic, target_generation): shiftable_forms = get_shiftable_forms(pfic) diff --git a/ui/workers/gather_evolutions_worker.py b/ui/workers/gather_evolutions_worker.py index 0f1da14..803a788 100644 --- a/ui/workers/gather_evolutions_worker.py +++ b/ui/workers/gather_evolutions_worker.py @@ -29,7 +29,7 @@ class GatherEvolutions(QRunnable): except Exception as e: print(f"Error gathering Pokémon home storage status: {e}") - def gather_evolution_data(self, force_refresh = True): + def gather_evolution_data(self, force_refresh = False): all_pokemon_forms = db.get_list_of_pokemon_forms() evolutions = {} @@ -135,7 +135,7 @@ class GatherEvolutions(QRunnable): evolutions[composite_key] = (evolution_info) self.traverse_and_store(next_stage, evolutions, gender) - def parse_evolution_chain(self, table, pokemon_form, force_refresh = True): + def parse_evolution_chain(self, table, pokemon_form, force_refresh = False): cache_record_name = f"evo_{pokemon_form['pfic']}" if force_refresh: cache.purge(cache_record_name) diff --git a/ui/workers/gather_pokemon_forms_worker.py b/ui/workers/gather_pokemon_forms_worker.py index fe2d690..444bf03 100644 --- a/ui/workers/gather_pokemon_forms_worker.py +++ b/ui/workers/gather_pokemon_forms_worker.py @@ -62,7 +62,7 @@ class GatherPokemonFormsWorker(QRunnable): return form_name return "None" - def process_pokemon_entry(self, national_dex_number, pokemon_soup, force_refresh = True): + def process_pokemon_entry(self, national_dex_number, pokemon_soup, force_refresh = False): found_forms = [] generation = get_generation_from_national_dex(national_dex_number) pokemon_name = pokemon_soup.get_text(strip=True) -- 2.30.2