7 changed files with 702 additions and 177 deletions
@ -0,0 +1,49 @@ |
|||||
|
[ |
||||
|
"Male", |
||||
|
"Normal Forme", |
||||
|
"Hero of Many Battles", |
||||
|
"Altered Forme", |
||||
|
"Land Forme", |
||||
|
"Standard Mode", |
||||
|
"Galarian Standard Mode", |
||||
|
"Ordinary Forme", |
||||
|
"Aria Forme", |
||||
|
"Natural Form", |
||||
|
"Shield Forme", |
||||
|
"Neutral Mode", |
||||
|
"Hoopa Confined", |
||||
|
"Solo Form", |
||||
|
"Type: Normal", |
||||
|
"Red Core", |
||||
|
"Disguised Form", |
||||
|
"Ice Face", |
||||
|
"Full Belly Mode", |
||||
|
"Zero Form", |
||||
|
"Curly Form", |
||||
|
"Chest Form", |
||||
|
"Apex Build", |
||||
|
"Ultimate Mode", |
||||
|
"Teal Mask", |
||||
|
"Normal Form", |
||||
|
"Plant Cloak", |
||||
|
"Overcast Form", |
||||
|
"West Sea", |
||||
|
"Normal", |
||||
|
"Red-Striped Form", |
||||
|
"Spring Form", |
||||
|
"Incarnate Forme", |
||||
|
"Meadow Pattern", |
||||
|
"Red Flower", |
||||
|
"Average Size", |
||||
|
"50% Forme", |
||||
|
"Confined", |
||||
|
"Baile Style", |
||||
|
"Midday Form", |
||||
|
"Amped Form", |
||||
|
"Vanilla Cream Strawberry Sweet", |
||||
|
"Single Strike Style", |
||||
|
"Family of Three", |
||||
|
"Green Plumage", |
||||
|
"Two-Segment Form", |
||||
|
"Standard Form" |
||||
|
] |
||||
@ -0,0 +1,186 @@ |
|||||
|
import json |
||||
|
import sqlite3 |
||||
|
from cache_manager import CacheManager |
||||
|
from bs4 import BeautifulSoup, Tag |
||||
|
|
||||
|
def create_pokemon_storage_db(): |
||||
|
conn = sqlite3.connect('pokemon_forms.db') |
||||
|
cursor = conn.cursor() |
||||
|
cursor.execute(''' |
||||
|
CREATE TABLE IF NOT EXISTS pokemon_storage ( |
||||
|
PFIC TEXT PRIMARY KEY, |
||||
|
storable_in_home BOOLEAN NOT NULL, |
||||
|
FOREIGN KEY (PFIC) REFERENCES pokemon_forms (PFIC) |
||||
|
) |
||||
|
''') |
||||
|
conn.commit() |
||||
|
return conn |
||||
|
|
||||
|
def insert_pokemon_storage(conn, pfic: str, storable_in_home: bool): |
||||
|
cursor = conn.cursor() |
||||
|
cursor.execute(''' |
||||
|
INSERT OR REPLACE INTO pokemon_storage |
||||
|
(PFIC, storable_in_home) |
||||
|
VALUES (?, ?) |
||||
|
''', (pfic, storable_in_home)) |
||||
|
conn.commit() |
||||
|
|
||||
|
def scrape_serebii_region_pokemon(url, cache): |
||||
|
response = cache.fetch_url(url) |
||||
|
|
||||
|
if not response: |
||||
|
return [] |
||||
|
|
||||
|
soup = BeautifulSoup(response, 'html.parser') |
||||
|
|
||||
|
pokemon_list = [] |
||||
|
|
||||
|
# Find the main table containing Pokémon data |
||||
|
table = soup.find('table', class_='dextable') |
||||
|
|
||||
|
if table: |
||||
|
rows = table.find_all('tr')[2:] # Skip the header row and the game intro row |
||||
|
for row in rows: |
||||
|
cells = row.find_all('td') |
||||
|
if len(cells) <= 5: # Ensure we have enough cells to check depositability. if only 5 then its not depositable in any game. |
||||
|
continue |
||||
|
|
||||
|
number = cells[0].text.strip().lstrip('#') |
||||
|
name = cells[2].text.strip() |
||||
|
|
||||
|
# Get the image URL |
||||
|
img_url = cells[1].find('img')['src'] |
||||
|
full_img_url = f"https://www.serebii.net{img_url}" |
||||
|
|
||||
|
pokemon_list.append({ |
||||
|
'number': number, |
||||
|
'name': name, |
||||
|
'image_url': full_img_url |
||||
|
}) |
||||
|
|
||||
|
return pokemon_list |
||||
|
|
||||
|
def scrape_all_regions(cache): |
||||
|
base_url = "https://www.serebii.net/pokemonhome/" |
||||
|
regions = ["kanto", "johto", "hoenn", "sinnoh", "unova", "kalos", "alola", "galar", "paldea", "hisui", "unknown"] |
||||
|
all_pokemon = [] |
||||
|
|
||||
|
for region in regions: |
||||
|
url = f"{base_url}{region}pokemon.shtml" |
||||
|
region_pokemon = scrape_serebii_region_pokemon(url, cache) |
||||
|
all_pokemon.extend(region_pokemon) |
||||
|
print(f"Scraped {len(region_pokemon)} Pokémon from {region.capitalize()} region") |
||||
|
|
||||
|
return all_pokemon |
||||
|
|
||||
|
def get_objects_by_number(array, target_number): |
||||
|
return [obj for obj in array if obj['number'] == target_number] |
||||
|
|
||||
|
def extract_bracketed_text(string): |
||||
|
results = [] |
||||
|
stack = [] |
||||
|
start_index = -1 |
||||
|
|
||||
|
for i, char in enumerate(string): |
||||
|
if char == '(': |
||||
|
if not stack: |
||||
|
start_index = i |
||||
|
stack.append(i) |
||||
|
elif char == ')': |
||||
|
if stack: |
||||
|
stack.pop() |
||||
|
if not stack: |
||||
|
results.append(string[start_index + 1:i]) |
||||
|
start_index = -1 |
||||
|
else: |
||||
|
print(f"Warning: Unmatched closing parenthesis at position {i}") |
||||
|
|
||||
|
# Handle any remaining unclosed brackets |
||||
|
if stack: |
||||
|
print(f"Warning: {len(stack)} unmatched opening parentheses") |
||||
|
for unmatched_start in stack: |
||||
|
results.append(string[unmatched_start + 1:]) |
||||
|
|
||||
|
return results |
||||
|
|
||||
|
def compare_forms(a, b): |
||||
|
if a == b: |
||||
|
return True |
||||
|
|
||||
|
temp_a = a.lower().replace("forme", "").replace("form", "").replace("é", "e").strip() |
||||
|
temp_b = b.lower().replace("forme", "").replace("form", "").replace("é", "e").strip() |
||||
|
|
||||
|
temp_a = temp_a.replace("deputante", "debutante").replace("p'au", "pa'u").replace("blood moon", "bloodmoon") |
||||
|
temp_b = temp_b.replace("deputante", "debutante").replace("p'au", "pa'u").replace("blood moon", "bloodmoon") |
||||
|
|
||||
|
if temp_a == temp_b: |
||||
|
return True |
||||
|
|
||||
|
return False |
||||
|
|
||||
|
if __name__ == "__main__": |
||||
|
cache = CacheManager() |
||||
|
|
||||
|
conn = create_pokemon_storage_db() |
||||
|
cursor = conn.cursor() |
||||
|
cursor.execute(''' |
||||
|
SELECT pf.PFIC, pf.name, pf.form_name, pf.national_dex |
||||
|
FROM pokemon_forms pf |
||||
|
ORDER BY pf.national_dex, pf.form_name |
||||
|
''') |
||||
|
pokemon_forms = cursor.fetchall() |
||||
|
|
||||
|
all_depositable_pokemon = scrape_all_regions(cache) |
||||
|
|
||||
|
try: |
||||
|
with open('./DataGatherers/DefaultForms.json', 'r') as f: |
||||
|
default_forms = json.load(f) |
||||
|
except FileNotFoundError: |
||||
|
default_forms = [] |
||||
|
|
||||
|
for pfic, name, form, national_dex in pokemon_forms: |
||||
|
print(f"Processing {name} {form if form else ''}") |
||||
|
|
||||
|
storable_in_home = False |
||||
|
|
||||
|
if form and name in form: |
||||
|
form = form.replace(name, "").strip() |
||||
|
|
||||
|
# serebii doesn't list gender in the table so we have to assume based on form name. |
||||
|
if form and ("male" in form.lower() or "female" in form.lower()): |
||||
|
form = None |
||||
|
|
||||
|
if form and form in default_forms: |
||||
|
form = None |
||||
|
|
||||
|
if name == "Unown" and (form != "!" and form != "?"): |
||||
|
form = None |
||||
|
|
||||
|
if name == "Tauros" and form == "Combat Breed": |
||||
|
form = "Paldean Form" |
||||
|
|
||||
|
if name == "Alcremie": |
||||
|
form = None |
||||
|
|
||||
|
pokemon = get_objects_by_number(all_depositable_pokemon, f"{national_dex:04d}") |
||||
|
for p in pokemon: |
||||
|
if form == None and name.lower() in p['name'].lower(): |
||||
|
storable_in_home = True |
||||
|
break |
||||
|
|
||||
|
parts = p['name'].split(" ") |
||||
|
if len(parts) > 1 and parts[0] == form: |
||||
|
storable_in_home = True |
||||
|
|
||||
|
brackets = extract_bracketed_text(p['name']) |
||||
|
if brackets: |
||||
|
for bracket in brackets: |
||||
|
if name in bracket: |
||||
|
bracket = bracket.replace(name, "").strip() |
||||
|
if compare_forms(form, bracket): |
||||
|
storable_in_home = True |
||||
|
break |
||||
|
|
||||
|
print(f"{name} {form if form else ''} is storable in home: {storable_in_home}") |
||||
|
insert_pokemon_storage(conn, pfic, storable_in_home) |
||||
|
|
||||
@ -1,131 +1 @@ |
|||||
{ |
{"evolution_0019-01-000-2_0020-01-001-0": {"action": "delete"}, "evolution_0868-08-000-0_0869-08-001-0": {"action": "update", "new_from_pfic": "0868-08-000-0", "new_to_pfic": "0869-08-001-0", "new_method": "Spin clockwise for more than 5 seconds during the day while holding a Berry Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-002-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-002-0", "method": "Spin clockwise for more than 5 seconds during the day while holding a Clover Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-003-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-003-0", "method": "Spin clockwise for more than 5 seconds during the day while holding a Flower Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-004-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-004-0", "method": "Spin clockwise for more than 5 seconds during the day while holding a Love Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-005-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-005-0", "method": "Spin clockwise for more than 5 seconds during the day while holding a Ribbon Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-006-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-006-0", "method": "Spin clockwise for more than 5 seconds during the day while holding a Star Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-007-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-007-0", "method": "Spin clockwise for more than 5 seconds during the day while holding a Strawberry Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-008-0": {"action": "update", "new_from_pfic": "0868-08-000-0", "new_to_pfic": "0869-08-008-0", "new_method": "Spin clockwise for more than 5 seconds at night while holding a Berry Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-009-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-009-0", "method": "Spin clockwise for more than 5 seconds at night while holding a Clover Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-010-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-010-0", "method": "Spin clockwise for more than 5 seconds at night while holding a Flower Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-011-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-011-0", "method": "Spin clockwise for more than 5 seconds at night while holding a Love Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-012-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-012-0", "method": "Spin clockwise for more than 5 seconds at night while holding a Ribbon Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-013-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-013-0", "method": "Spin clockwise for more than 5 seconds at night while holding a Star Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-014-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-014-0", "method": "Spin clockwise for more than 5 seconds at night while holding a Strawberry Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-016-0": {"action": "update", "new_from_pfic": "0868-08-000-0", "new_to_pfic": "0869-08-016-0", "new_method": "Spin clockwise for less than 5 seconds at night while holding a Berry Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-017-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-017-0", "method": "Spin clockwise for less than 5 seconds at night while holding a Clover Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-018-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-018-0", "method": "Spin clockwise for less than 5 seconds at night while holding a Flower Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-019-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-019-0", "method": "Spin clockwise for less than 5 seconds at night while holding a Love Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-020-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-020-0", "method": "Spin clockwise for less than 5 seconds at night while holding a Ribbon Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-021-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-021-0", "method": "Spin clockwise for less than 5 seconds at night while holding a Star Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-022-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-022-0", "method": "Spin clockwise for less than 5 seconds at night while holding a Strawberry Sweet \u2192"}, "0869-08-064-0": {"storable_in_home": false}} |
||||
"evolution_0019-01-000-2_0020-01-001-0": { |
|
||||
"action": "delete" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-001-0": { |
|
||||
"action": "update", |
|
||||
"new_from_pfic": "0868-08-000-0", |
|
||||
"new_to_pfic": "0869-08-001-0", |
|
||||
"new_method": "Spin clockwise for more than 5 seconds during the day while holding a Berry Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-002-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-002-0", |
|
||||
"method": "Spin clockwise for more than 5 seconds during the day while holding a Clover Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-003-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-003-0", |
|
||||
"method": "Spin clockwise for more than 5 seconds during the day while holding a Flower Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-004-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-004-0", |
|
||||
"method": "Spin clockwise for more than 5 seconds during the day while holding a Love Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-005-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-005-0", |
|
||||
"method": "Spin clockwise for more than 5 seconds during the day while holding a Ribbon Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-006-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-006-0", |
|
||||
"method": "Spin clockwise for more than 5 seconds during the day while holding a Star Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-007-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-007-0", |
|
||||
"method": "Spin clockwise for more than 5 seconds during the day while holding a Strawberry Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-008-0": { |
|
||||
"action": "update", |
|
||||
"new_from_pfic": "0868-08-000-0", |
|
||||
"new_to_pfic": "0869-08-008-0", |
|
||||
"new_method": "Spin clockwise for more than 5 seconds at night while holding a Berry Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-009-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-009-0", |
|
||||
"method": "Spin clockwise for more than 5 seconds at night while holding a Clover Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-010-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-010-0", |
|
||||
"method": "Spin clockwise for more than 5 seconds at night while holding a Flower Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-011-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-011-0", |
|
||||
"method": "Spin clockwise for more than 5 seconds at night while holding a Love Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-012-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-012-0", |
|
||||
"method": "Spin clockwise for more than 5 seconds at night while holding a Ribbon Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-013-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-013-0", |
|
||||
"method": "Spin clockwise for more than 5 seconds at night while holding a Star Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-014-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-014-0", |
|
||||
"method": "Spin clockwise for more than 5 seconds at night while holding a Strawberry Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-016-0": { |
|
||||
"action": "update", |
|
||||
"new_from_pfic": "0868-08-000-0", |
|
||||
"new_to_pfic": "0869-08-016-0", |
|
||||
"new_method": "Spin clockwise for less than 5 seconds at night while holding a Berry Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-017-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-017-0", |
|
||||
"method": "Spin clockwise for less than 5 seconds at night while holding a Clover Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-018-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-018-0", |
|
||||
"method": "Spin clockwise for less than 5 seconds at night while holding a Flower Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-019-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-019-0", |
|
||||
"method": "Spin clockwise for less than 5 seconds at night while holding a Love Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-020-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-020-0", |
|
||||
"method": "Spin clockwise for less than 5 seconds at night while holding a Ribbon Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-021-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-021-0", |
|
||||
"method": "Spin clockwise for less than 5 seconds at night while holding a Star Sweet \u2192" |
|
||||
}, |
|
||||
"evolution_0868-08-000-0_0869-08-022-0": { |
|
||||
"action": "add", |
|
||||
"from_pfic": "0868-08-000-0", |
|
||||
"to_pfic": "0869-08-022-0", |
|
||||
"method": "Spin clockwise for less than 5 seconds at night while holding a Strawberry Sweet \u2192" |
|
||||
} |
|
||||
} |
|
||||
Binary file not shown.
Loading…
Reference in new issue