Added items and ability as gen data + updated data

MANIFEST.in

@@ -4,4 +4,6 @@ include src/poke_env/data/static/*.html
 include src/poke_env/data/static/moves/*.json
 include src/poke_env/data/static/pokedex/*.json
 include src/poke_env/data/static/typechart/*.json
+include src/poke_env/data/static/abilities/*.json
+include src/poke_env/data/static/items/*.json
 include src/poke_env/py.typed

scripts/data_script_utils.py

@@ -8,86 +8,99 @@
 MAX_MOVE_IDX_PER_GEN = {1: 165, 2: 251, 3: 354, 4: 467, 5: 559, 6: 621, 7: 742, 8: 850}
 STATIC_DATA_ROOT = "src/poke_env/data/static"
 
-
 def fetch_and_clean_ps_data(url: str, deserialize: bool = True):
     data = requests.get(url).text
-
     if data == "404: Not Found":
         return {}
 
-    # Remove start and end of the file
+    # Trim the JS wrapper to the main object
     data = "{" + "= {".join(data.split("= {")[1:])[:-2]
 
-    # Transform tabs into spaces
+    # Normalize whitespace early
     data = data.replace("\t", " ")
 
-    # Transform keys into correct json strings
-    data = re.sub(r"([\w\d]+): ", r'"\1": ', data)
-
-    # Transform single quoted text into double quoted text
-    data = re.sub(r"'([\w\d ]+)'", r'"\1"', data)
-
-    # Remove comments
-    data = re.sub(r" +//.+", "", data)
-
-    # Remove empty lines
+    # Strip comments (// and /* */)
+    data = re.sub(r" +//.*", "", data)
+    data = re.sub(r"/\*.*?\*/", "", data, flags=re.DOTALL)
+
+    # Helper: replace method shorthands like: name(params) { ... },
+    for n_space in range(14):
+        spaces = " " * n_space
+
+        # Multiline method shorthand
+        pattern_method = (
+            r"^" + spaces +
+            r"(\w+)\s*\(\s*[\s\S]*?\)\s*\{\n" +     # tolerate anything in (...)
+            r"(?:.*\n)*?" +                          # lazy body
+            spaces + r"\},"
+        )
+        sub = spaces + r'"\1": "\1",'
+        data = re.sub(pattern_method, sub, data, flags=re.MULTILINE)
+
+        # Empty/one-liners: name() {},
+        pattern_method_empty = r"^" + spaces + r"(\w+)\s*\(\s*[\s\S]*?\)\s*\{\s*\},"
+        data = re.sub(pattern_method_empty, sub, data, flags=re.MULTILINE)
+
+        # Property with function keyword: name: function (...) { ... },
+        pattern_fnkw = (
+            r"^" + spaces +
+            r'"?(\w+)"?\s*:\s*function\s*\(\s*[\s\S]*?\)\s*\{\n' +
+            r"(?:.*\n)*?" +
+            spaces + r"\},"
+        )
+        data = re.sub(pattern_fnkw, sub, data, flags=re.MULTILINE)
+
+        # Arrow function with block body: name: (...) => { ... },
+        pattern_arrow_block = (
+            r"^" + spaces +
+            r'"?(\w+)"?\s*:\s*\(\s*[\s\S]*?\)\s*=>\s*\{\n' +
+            r"(?:.*\n)*?" +
+            spaces + r"\},"
+        )
+        data = re.sub(pattern_arrow_block, sub, data, flags=re.MULTILINE)
+
+        # Arrow function concise body: name: (...) => expr,
+        pattern_arrow_expr = (
+            r"^" + spaces +
+            r'"?(\w+)"?\s*:\s*\(\s*[\s\S]*?\)\s*=>\s*(?:[^,\n]|\n(?!' + spaces + r'\S))*?,'
+        )
+        data = re.sub(pattern_arrow_expr, sub, data, flags=re.MULTILINE)
+
+    # Empty callbacks like onX() {}
+    data = re.sub(r"(\bon\w+\b|\b\w+Callback\b)\(\s*\)\s*\{\s*\}", r'"\1": "\1"', data)
+
+    #Normalize the literal: () => null
+    data = re.sub(r"\(\s*\)\s*=>\s*null", r"null", data)
+
+    # Key/quote cleanup
+    data = re.sub(r"([\w\d]+): ", r'"\1": ', data)              # quote bare keys
+    data = re.sub(r"'([^'\n]+)'", r'"\1"', data)                # single → double
+
+    # Remove extra blank lines / trailing commas
     for _ in range(3):
         data = re.sub(r"\n\n", "\n", data)
-
     data = re.sub(r",\n( +)\]", r"\n\1]", data)
-
-    # Correct double-quoted text inside double-quoted text
-    data = re.sub(r': ""(.*)":(.*)",', r': "\1:\2",', data)
-
-    # Correct isolated "undefined" values
-    data = re.sub(r": undefined", r": null", data)
-
-    # Callback and handlers
-    for function_title_match in (r"(on\w+)", r"(\w+Callback)"):
-        for n_space in range(10):
-            spaces = " " * (n_space)
-            pattern = (
-                r"^"
-                + spaces
-                + function_title_match
-                + r"\((\w+, )*(\w+)?\) \{\n(.+\n)+?"
-                + spaces
-                + r"\},"
-            )
-            sub = spaces + r'"\1": "\1",'
-            data = re.sub(pattern, sub, data, flags=re.MULTILINE)
-        pattern = function_title_match + r"\(\) \{\s*\}"
-        sub = r'"\1": "\1"'
-        data = re.sub(pattern, sub, data, flags=re.MULTILINE)
-
-    # Remove incorrect commas
     data = re.sub(r",\n( *)\}", r"\n\1}", data)
 
-    # Null arrow functions
-    data = re.sub(r"\(\) => null", r"null", data)
-
-    # Remove incorrect commas
-    data = re.sub(r",\n( *)\}", r"\n\1}", data)
-    data = re.sub(r",\n( +)\]", r"\n\1]", data)
-    # Correct double-quoted text inside double-quoted text
-
+    # Fix tricky embedded quotes
+    data = re.sub(r': ""(.*)":(.*)",', r': "\1:\2",', data)
     data = re.sub(r': "(.*)"(.*)":(.*)",', r': "\1\2:\3",', data)
     data = re.sub(r': ""(.*)":(.*)",', r': "\1:\2",', data)
 
-    # Correct non-quoted number keys
+    # Misc normalizations
+    data = re.sub(r": undefined", r": null", data)
     data = re.sub(r"(\d+):", r'"\1":', data)
-    # Correct non-quoted H keys
-
-    data = re.sub(r"H: ", r'"H": ', data)
+    data = re.sub(r"\bH: ", r'"H": ', data)
     data = re.sub(r", moves:", r', "moves":', data)
     data = re.sub(r", nature:", r', "nature":', data)
 
+    # Final trailing-comma cleanup
+    data = re.sub(r",\n( *)\}", r"\n\1}", data)
+    data = re.sub(r",\n( +)\]", r"\n\1]", data)
+
     try:
-        if deserialize:
-            return json.loads(data)
-        else:
-            return data
-    except Exception:
+        return json.loads(data) if deserialize else data
+    except Exception as e:
         with open("out.json", "w+") as f:
             f.write(data)
-        raise Exception
+        raise

scripts/update_abilities.py

@@ -0,0 +1,22 @@
+import json
+
+from data_script_utils import (
+    CURRENT_GEN,
+    STATIC_DATA_ROOT,
+    fetch_and_clean_ps_data,
+)
+
+data_by_gen = {
+    CURRENT_GEN: fetch_and_clean_ps_data(
+        "https://raw.githubusercontent.com/smogon/pokemon-showdown/master/data/abilities.ts"
+    )
+}
+
+for gen in range(1, CURRENT_GEN):
+    data_by_gen[gen] = fetch_and_clean_ps_data(
+        f"https://raw.githubusercontent.com/smogon/pokemon-showdown/master/data/mods/gen{gen}/abilities.ts"
+    )
+
+for gen in range(CURRENT_GEN, 0, -1):
+    with open(f"{STATIC_DATA_ROOT}/abilities/gen{gen}abilities.json", "w+") as f:
+        f.write(json.dumps(data_by_gen[gen], indent=4, sort_keys=True))

scripts/update_all_data.sh

@@ -1,3 +1,5 @@
 python scripts/update_learnset.py
 python scripts/update_moves.py
 python scripts/update_pokedex.py
+python scripts/update_items.py
+python scripts/update_abilities.py

scripts/update_items.py

@@ -0,0 +1,22 @@
+import json
+
+from data_script_utils import (
+    CURRENT_GEN,
+    STATIC_DATA_ROOT,
+    fetch_and_clean_ps_data,
+)
+
+data_by_gen = {
+    CURRENT_GEN: fetch_and_clean_ps_data(
+        "https://raw.githubusercontent.com/smogon/pokemon-showdown/master/data/items.ts"
+    )
+}
+
+for gen in range(1, CURRENT_GEN):
+    data_by_gen[gen] = fetch_and_clean_ps_data(
+        f"https://raw.githubusercontent.com/smogon/pokemon-showdown/master/data/mods/gen{gen}/items.ts"
+    )
+
+for gen in range(CURRENT_GEN, 0, -1):
+    with open(f"{STATIC_DATA_ROOT}/items/gen{gen}items.json", "w+") as f:
+        f.write(json.dumps(data_by_gen[gen], indent=4, sort_keys=True))

src/poke_env/data/gen_data.py

@@ -10,7 +10,7 @@
 
 
 class GenData:
-    __slots__ = ("gen", "moves", "natures", "pokedex", "type_chart", "learnset")
+    __slots__ = ("gen", "moves", "natures", "pokedex", "type_chart", "learnset", "abilities", "items")
 
     UNKNOWN_ITEM = "unknown_item"
 
@@ -26,6 +26,8 @@ def __init__(self, gen: int):
         self.pokedex = self.load_pokedex(gen)
         self.type_chart = self.load_type_chart(gen)
         self.learnset = self.load_learnset()
+        self.abilities = self.load_abilities(gen)
+        self.items = self.load_items(gen)
 
     def __deepcopy__(self, memodict: Optional[Dict[int, Any]] = None) -> GenData:
         return self
@@ -44,6 +46,15 @@ def load_learnset(self) -> Dict[str, Dict[str, Union[int, float]]]:
         with open(os.path.join(self._static_files_root, "learnset.json")) as f:
             return orjson.loads(f.read())
 
+    def load_abilities(self, gen: int) -> Dict[str, Dict[str, Union[int, float]]]:
+        with open(os.path.join(self._static_files_root, "abilities", f"gen{gen}abilities.json")) as f:
+            return orjson.loads(f.read())
+
+    def load_items(self, gen: int) -> Dict[str, Dict[str, Union[int, float]]]:
+        with open(os.path.join(self._static_files_root, "items", f"gen{gen}items.json")) as f:
+            return orjson.loads(f.read())
+
+
     def load_pokedex(self, gen: int) -> Dict[str, Any]:
         with open(
             os.path.join(self._static_files_root, "pokedex", f"gen{gen}pokedex.json")

src/poke_env/data/static/abilities/gen1abilities.json

@@ -0,0 +1 @@
+{}

src/poke_env/data/static/abilities/gen2abilities.json

@@ -0,0 +1 @@
+{}

src/poke_env/data/static/abilities/gen3abilities.json

@@ -0,0 +1,92 @@
+{
+    "cutecharm": {
+        "inherit": true,
+        "onDamagingHit": "onDamagingHit"
+    },
+    "effectspore": {
+        "inherit": true,
+        "onDamagingHit": "onDamagingHit"
+    },
+    "flamebody": {
+        "inherit": true,
+        "onDamagingHit": "onDamagingHit"
+    },
+    "flashfire": {
+        "inherit": true,
+        "onTryHit": "onTryHit"
+    },
+    "forecast": {
+        "flags": {},
+        "inherit": true
+    },
+    "hustle": {
+        "inherit": true,
+        "onSourceModifyAccuracy": "onSourceModifyAccuracy"
+    },
+    "intimidate": {
+        "inherit": true,
+        "onStart": "onStart"
+    },
+    "lightningrod": {
+        "flags": {
+            "breakable": 1
+        },
+        "name": "Lightning Rod",
+        "num": 32,
+        "onFoeRedirectTarget": "onFoeRedirectTarget",
+        "rating": 0
+    },
+    "magnetpull": {
+        "inherit": true,
+        "onAnyMaybeTrapPokemon": "onAnyMaybeTrapPokemon",
+        "onFoeTrapPokemon": "onFoeTrapPokemon"
+    },
+    "minus": {
+        "inherit": true,
+        "onModifySpA": "onModifySpA"
+    },
+    "plus": {
+        "inherit": true,
+        "onModifySpA": "onModifySpA"
+    },
+    "poisonpoint": {
+        "inherit": true,
+        "onDamagingHit": "onDamagingHit"
+    },
+    "pressure": {
+        "inherit": true,
+        "onStart": "onStart"
+    },
+    "raindish": {
+        "inherit": true,
+        "onWeather": "onWeather"
+    },
+    "roughskin": {
+        "inherit": true,
+        "onDamagingHit": "onDamagingHit"
+    },
+    "shadowtag": {
+        "inherit": true,
+        "onFoeTrapPokemon": "onFoeTrapPokemon"
+    },
+    "static": {
+        "inherit": true,
+        "onDamagingHit": "onDamagingHit"
+    },
+    "trace": {
+        "flags": {},
+        "inherit": true,
+        "onUpdate": "onUpdate"
+    },
+    "truant": {
+        "inherit": true,
+        "onBeforeMove": "onBeforeMove",
+        "onResidual": "onResidual",
+        "onResidualOrder": 27,
+        "onStart": "onStart"
+    },
+    "voltabsorb": {
+        "inherit": true,
+        "onTryHit": "onTryHit"
+    }
+}