7 changed files with 702 additions and 177 deletions
@ -0,0 +1,49 @@ |
|||
[ |
|||
"Male", |
|||
"Normal Forme", |
|||
"Hero of Many Battles", |
|||
"Altered Forme", |
|||
"Land Forme", |
|||
"Standard Mode", |
|||
"Galarian Standard Mode", |
|||
"Ordinary Forme", |
|||
"Aria Forme", |
|||
"Natural Form", |
|||
"Shield Forme", |
|||
"Neutral Mode", |
|||
"Hoopa Confined", |
|||
"Solo Form", |
|||
"Type: Normal", |
|||
"Red Core", |
|||
"Disguised Form", |
|||
"Ice Face", |
|||
"Full Belly Mode", |
|||
"Zero Form", |
|||
"Curly Form", |
|||
"Chest Form", |
|||
"Apex Build", |
|||
"Ultimate Mode", |
|||
"Teal Mask", |
|||
"Normal Form", |
|||
"Plant Cloak", |
|||
"Overcast Form", |
|||
"West Sea", |
|||
"Normal", |
|||
"Red-Striped Form", |
|||
"Spring Form", |
|||
"Incarnate Forme", |
|||
"Meadow Pattern", |
|||
"Red Flower", |
|||
"Average Size", |
|||
"50% Forme", |
|||
"Confined", |
|||
"Baile Style", |
|||
"Midday Form", |
|||
"Amped Form", |
|||
"Vanilla Cream Strawberry Sweet", |
|||
"Single Strike Style", |
|||
"Family of Three", |
|||
"Green Plumage", |
|||
"Two-Segment Form", |
|||
"Standard Form" |
|||
] |
|||
@ -0,0 +1,186 @@ |
|||
import json |
|||
import sqlite3 |
|||
from cache_manager import CacheManager |
|||
from bs4 import BeautifulSoup, Tag |
|||
|
|||
def create_pokemon_storage_db(): |
|||
conn = sqlite3.connect('pokemon_forms.db') |
|||
cursor = conn.cursor() |
|||
cursor.execute(''' |
|||
CREATE TABLE IF NOT EXISTS pokemon_storage ( |
|||
PFIC TEXT PRIMARY KEY, |
|||
storable_in_home BOOLEAN NOT NULL, |
|||
FOREIGN KEY (PFIC) REFERENCES pokemon_forms (PFIC) |
|||
) |
|||
''') |
|||
conn.commit() |
|||
return conn |
|||
|
|||
def insert_pokemon_storage(conn, pfic: str, storable_in_home: bool): |
|||
cursor = conn.cursor() |
|||
cursor.execute(''' |
|||
INSERT OR REPLACE INTO pokemon_storage |
|||
(PFIC, storable_in_home) |
|||
VALUES (?, ?) |
|||
''', (pfic, storable_in_home)) |
|||
conn.commit() |
|||
|
|||
def scrape_serebii_region_pokemon(url, cache): |
|||
response = cache.fetch_url(url) |
|||
|
|||
if not response: |
|||
return [] |
|||
|
|||
soup = BeautifulSoup(response, 'html.parser') |
|||
|
|||
pokemon_list = [] |
|||
|
|||
# Find the main table containing Pokémon data |
|||
table = soup.find('table', class_='dextable') |
|||
|
|||
if table: |
|||
rows = table.find_all('tr')[2:] # Skip the header row and the game intro row |
|||
for row in rows: |
|||
cells = row.find_all('td') |
|||
if len(cells) <= 5: # Ensure we have enough cells to check depositability. if only 5 then its not depositable in any game. |
|||
continue |
|||
|
|||
number = cells[0].text.strip().lstrip('#') |
|||
name = cells[2].text.strip() |
|||
|
|||
# Get the image URL |
|||
img_url = cells[1].find('img')['src'] |
|||
full_img_url = f"https://www.serebii.net{img_url}" |
|||
|
|||
pokemon_list.append({ |
|||
'number': number, |
|||
'name': name, |
|||
'image_url': full_img_url |
|||
}) |
|||
|
|||
return pokemon_list |
|||
|
|||
def scrape_all_regions(cache): |
|||
base_url = "https://www.serebii.net/pokemonhome/" |
|||
regions = ["kanto", "johto", "hoenn", "sinnoh", "unova", "kalos", "alola", "galar", "paldea", "hisui", "unknown"] |
|||
all_pokemon = [] |
|||
|
|||
for region in regions: |
|||
url = f"{base_url}{region}pokemon.shtml" |
|||
region_pokemon = scrape_serebii_region_pokemon(url, cache) |
|||
all_pokemon.extend(region_pokemon) |
|||
print(f"Scraped {len(region_pokemon)} Pokémon from {region.capitalize()} region") |
|||
|
|||
return all_pokemon |
|||
|
|||
def get_objects_by_number(array, target_number): |
|||
return [obj for obj in array if obj['number'] == target_number] |
|||
|
|||
def extract_bracketed_text(string): |
|||
results = [] |
|||
stack = [] |
|||
start_index = -1 |
|||
|
|||
for i, char in enumerate(string): |
|||
if char == '(': |
|||
if not stack: |
|||
start_index = i |
|||
stack.append(i) |
|||
elif char == ')': |
|||
if stack: |
|||
stack.pop() |
|||
if not stack: |
|||
results.append(string[start_index + 1:i]) |
|||
start_index = -1 |
|||
else: |
|||
print(f"Warning: Unmatched closing parenthesis at position {i}") |
|||
|
|||
# Handle any remaining unclosed brackets |
|||
if stack: |
|||
print(f"Warning: {len(stack)} unmatched opening parentheses") |
|||
for unmatched_start in stack: |
|||
results.append(string[unmatched_start + 1:]) |
|||
|
|||
return results |
|||
|
|||
def compare_forms(a, b): |
|||
if a == b: |
|||
return True |
|||
|
|||
temp_a = a.lower().replace("forme", "").replace("form", "").replace("é", "e").strip() |
|||
temp_b = b.lower().replace("forme", "").replace("form", "").replace("é", "e").strip() |
|||
|
|||
temp_a = temp_a.replace("deputante", "debutante").replace("p'au", "pa'u").replace("blood moon", "bloodmoon") |
|||
temp_b = temp_b.replace("deputante", "debutante").replace("p'au", "pa'u").replace("blood moon", "bloodmoon") |
|||
|
|||
if temp_a == temp_b: |
|||
return True |
|||
|
|||
return False |
|||
|
|||
if __name__ == "__main__": |
|||
cache = CacheManager() |
|||
|
|||
conn = create_pokemon_storage_db() |
|||
cursor = conn.cursor() |
|||
cursor.execute(''' |
|||
SELECT pf.PFIC, pf.name, pf.form_name, pf.national_dex |
|||
FROM pokemon_forms pf |
|||
ORDER BY pf.national_dex, pf.form_name |
|||
''') |
|||
pokemon_forms = cursor.fetchall() |
|||
|
|||
all_depositable_pokemon = scrape_all_regions(cache) |
|||
|
|||
try: |
|||
with open('./DataGatherers/DefaultForms.json', 'r') as f: |
|||
default_forms = json.load(f) |
|||
except FileNotFoundError: |
|||
default_forms = [] |
|||
|
|||
for pfic, name, form, national_dex in pokemon_forms: |
|||
print(f"Processing {name} {form if form else ''}") |
|||
|
|||
storable_in_home = False |
|||
|
|||
if form and name in form: |
|||
form = form.replace(name, "").strip() |
|||
|
|||
# serebii doesn't list gender in the table so we have to assume based on form name. |
|||
if form and ("male" in form.lower() or "female" in form.lower()): |
|||
form = None |
|||
|
|||
if form and form in default_forms: |
|||
form = None |
|||
|
|||
if name == "Unown" and (form != "!" and form != "?"): |
|||
form = None |
|||
|
|||
if name == "Tauros" and form == "Combat Breed": |
|||
form = "Paldean Form" |
|||
|
|||
if name == "Alcremie": |
|||
form = None |
|||
|
|||
pokemon = get_objects_by_number(all_depositable_pokemon, f"{national_dex:04d}") |
|||
for p in pokemon: |
|||
if form == None and name.lower() in p['name'].lower(): |
|||
storable_in_home = True |
|||
break |
|||
|
|||
parts = p['name'].split(" ") |
|||
if len(parts) > 1 and parts[0] == form: |
|||
storable_in_home = True |
|||
|
|||
brackets = extract_bracketed_text(p['name']) |
|||
if brackets: |
|||
for bracket in brackets: |
|||
if name in bracket: |
|||
bracket = bracket.replace(name, "").strip() |
|||
if compare_forms(form, bracket): |
|||
storable_in_home = True |
|||
break |
|||
|
|||
print(f"{name} {form if form else ''} is storable in home: {storable_in_home}") |
|||
insert_pokemon_storage(conn, pfic, storable_in_home) |
|||
|
|||
@ -1,131 +1 @@ |
|||
{ |
|||
"evolution_0019-01-000-2_0020-01-001-0": { |
|||
"action": "delete" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-001-0": { |
|||
"action": "update", |
|||
"new_from_pfic": "0868-08-000-0", |
|||
"new_to_pfic": "0869-08-001-0", |
|||
"new_method": "Spin clockwise for more than 5 seconds during the day while holding a Berry Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-002-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-002-0", |
|||
"method": "Spin clockwise for more than 5 seconds during the day while holding a Clover Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-003-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-003-0", |
|||
"method": "Spin clockwise for more than 5 seconds during the day while holding a Flower Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-004-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-004-0", |
|||
"method": "Spin clockwise for more than 5 seconds during the day while holding a Love Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-005-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-005-0", |
|||
"method": "Spin clockwise for more than 5 seconds during the day while holding a Ribbon Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-006-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-006-0", |
|||
"method": "Spin clockwise for more than 5 seconds during the day while holding a Star Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-007-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-007-0", |
|||
"method": "Spin clockwise for more than 5 seconds during the day while holding a Strawberry Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-008-0": { |
|||
"action": "update", |
|||
"new_from_pfic": "0868-08-000-0", |
|||
"new_to_pfic": "0869-08-008-0", |
|||
"new_method": "Spin clockwise for more than 5 seconds at night while holding a Berry Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-009-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-009-0", |
|||
"method": "Spin clockwise for more than 5 seconds at night while holding a Clover Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-010-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-010-0", |
|||
"method": "Spin clockwise for more than 5 seconds at night while holding a Flower Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-011-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-011-0", |
|||
"method": "Spin clockwise for more than 5 seconds at night while holding a Love Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-012-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-012-0", |
|||
"method": "Spin clockwise for more than 5 seconds at night while holding a Ribbon Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-013-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-013-0", |
|||
"method": "Spin clockwise for more than 5 seconds at night while holding a Star Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-014-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-014-0", |
|||
"method": "Spin clockwise for more than 5 seconds at night while holding a Strawberry Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-016-0": { |
|||
"action": "update", |
|||
"new_from_pfic": "0868-08-000-0", |
|||
"new_to_pfic": "0869-08-016-0", |
|||
"new_method": "Spin clockwise for less than 5 seconds at night while holding a Berry Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-017-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-017-0", |
|||
"method": "Spin clockwise for less than 5 seconds at night while holding a Clover Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-018-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-018-0", |
|||
"method": "Spin clockwise for less than 5 seconds at night while holding a Flower Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-019-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-019-0", |
|||
"method": "Spin clockwise for less than 5 seconds at night while holding a Love Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-020-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-020-0", |
|||
"method": "Spin clockwise for less than 5 seconds at night while holding a Ribbon Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-021-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-021-0", |
|||
"method": "Spin clockwise for less than 5 seconds at night while holding a Star Sweet \u2192" |
|||
}, |
|||
"evolution_0868-08-000-0_0869-08-022-0": { |
|||
"action": "add", |
|||
"from_pfic": "0868-08-000-0", |
|||
"to_pfic": "0869-08-022-0", |
|||
"method": "Spin clockwise for less than 5 seconds at night while holding a Strawberry Sweet \u2192" |
|||
} |
|||
} |
|||
{"evolution_0019-01-000-2_0020-01-001-0": {"action": "delete"}, "evolution_0868-08-000-0_0869-08-001-0": {"action": "update", "new_from_pfic": "0868-08-000-0", "new_to_pfic": "0869-08-001-0", "new_method": "Spin clockwise for more than 5 seconds during the day while holding a Berry Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-002-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-002-0", "method": "Spin clockwise for more than 5 seconds during the day while holding a Clover Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-003-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-003-0", "method": "Spin clockwise for more than 5 seconds during the day while holding a Flower Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-004-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-004-0", "method": "Spin clockwise for more than 5 seconds during the day while holding a Love Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-005-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-005-0", "method": "Spin clockwise for more than 5 seconds during the day while holding a Ribbon Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-006-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-006-0", "method": "Spin clockwise for more than 5 seconds during the day while holding a Star Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-007-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-007-0", "method": "Spin clockwise for more than 5 seconds during the day while holding a Strawberry Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-008-0": {"action": "update", "new_from_pfic": "0868-08-000-0", "new_to_pfic": "0869-08-008-0", "new_method": "Spin clockwise for more than 5 seconds at night while holding a Berry Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-009-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-009-0", "method": "Spin clockwise for more than 5 seconds at night while holding a Clover Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-010-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-010-0", "method": "Spin clockwise for more than 5 seconds at night while holding a Flower Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-011-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-011-0", "method": "Spin clockwise for more than 5 seconds at night while holding a Love Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-012-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-012-0", "method": "Spin clockwise for more than 5 seconds at night while holding a Ribbon Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-013-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-013-0", "method": "Spin clockwise for more than 5 seconds at night while holding a Star Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-014-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-014-0", "method": "Spin clockwise for more than 5 seconds at night while holding a Strawberry Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-016-0": {"action": "update", "new_from_pfic": "0868-08-000-0", "new_to_pfic": "0869-08-016-0", "new_method": "Spin clockwise for less than 5 seconds at night while holding a Berry Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-017-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-017-0", "method": "Spin clockwise for less than 5 seconds at night while holding a Clover Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-018-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-018-0", "method": "Spin clockwise for less than 5 seconds at night while holding a Flower Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-019-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-019-0", "method": "Spin clockwise for less than 5 seconds at night while holding a Love Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-020-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-020-0", "method": "Spin clockwise for less than 5 seconds at night while holding a Ribbon Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-021-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-021-0", "method": "Spin clockwise for less than 5 seconds at night while holding a Star Sweet \u2192"}, "evolution_0868-08-000-0_0869-08-022-0": {"action": "add", "from_pfic": "0868-08-000-0", "to_pfic": "0869-08-022-0", "method": "Spin clockwise for less than 5 seconds at night while holding a Strawberry Sweet \u2192"}, "0869-08-064-0": {"storable_in_home": false}} |
|||
Binary file not shown.
Loading…
Reference in new issue