Merge main updates into FM

BaseClasses.py

@@ -869,21 +869,40 @@ def sweep_for_advancements(self, locations: Optional[Iterable[Location]] = None)
     def has(self, item: str, player: int, count: int = 1) -> bool:
         return self.prog_items[player][item] >= count
 
+    # for loops are specifically used in all/any/count methods, instead of all()/any()/sum(), to avoid the overhead of
+    # creating and iterating generator instances. In `return all(player_prog_items[item] for item in items)`, the
+    # argument to all() would be a new generator instance, for example.
     def has_all(self, items: Iterable[str], player: int) -> bool:
         """Returns True if each item name of items is in state at least once."""
-        return all(self.prog_items[player][item] for item in items)
+        player_prog_items = self.prog_items[player]
+        for item in items:
+            if not player_prog_items[item]:
+                return False
+        return True
 
     def has_any(self, items: Iterable[str], player: int) -> bool:
         """Returns True if at least one item name of items is in state at least once."""
-        return any(self.prog_items[player][item] for item in items)
+        player_prog_items = self.prog_items[player]
+        for item in items:
+            if player_prog_items[item]:
+                return True
+        return False
 
     def has_all_counts(self, item_counts: Mapping[str, int], player: int) -> bool:
         """Returns True if each item name is in the state at least as many times as specified."""
-        return all(self.prog_items[player][item] >= count for item, count in item_counts.items())
+        player_prog_items = self.prog_items[player]
+        for item, count in item_counts.items():
+            if player_prog_items[item] < count:
+                return False
+        return True
 
     def has_any_count(self, item_counts: Mapping[str, int], player: int) -> bool:
         """Returns True if at least one item name is in the state at least as many times as specified."""
-        return any(self.prog_items[player][item] >= count for item, count in item_counts.items())
+        player_prog_items = self.prog_items[player]
+        for item, count in item_counts.items():
+            if player_prog_items[item] >= count:
+                return True
+        return False
 
     def count(self, item: str, player: int) -> int:
         return self.prog_items[player][item]
@@ -911,11 +930,20 @@ def has_from_list_unique(self, items: Iterable[str], player: int, count: int) ->
 
     def count_from_list(self, items: Iterable[str], player: int) -> int:
         """Returns the cumulative count of items from a list present in state."""
-        return sum(self.prog_items[player][item_name] for item_name in items)
+        player_prog_items = self.prog_items[player]
+        total = 0
+        for item_name in items:
+            total += player_prog_items[item_name]
+        return total
 
     def count_from_list_unique(self, items: Iterable[str], player: int) -> int:
         """Returns the cumulative count of items from a list present in state. Ignores duplicates of the same item."""
-        return sum(self.prog_items[player][item_name] > 0 for item_name in items)
+        player_prog_items = self.prog_items[player]
+        total = 0
+        for item_name in items:
+            if player_prog_items[item_name] > 0:
+                total += 1
+        return total
 
     # item name group related
     def has_group(self, item_name_group: str, player: int, count: int = 1) -> bool:

MultiServer.py

@@ -28,9 +28,11 @@
 
 if typing.TYPE_CHECKING:
     import ssl
+    from NetUtils import ServerConnection
 
-import websockets
 import colorama
+import websockets
+from websockets.extensions.permessage_deflate import PerMessageDeflate
 try:
     # ponyorm is a requirement for webhost, not default server, so may not be importable
     from pony.orm.dbapiprovider import OperationalError
@@ -119,13 +121,14 @@ def get_saving_second(seed_name: str, interval: int = 60) -> int:
 
 class Client(Endpoint):
     version = Version(0, 0, 0)
-    tags: typing.List[str] = []
+    tags: typing.List[str]
     remote_items: bool
     remote_start_inventory: bool
     no_items: bool
     no_locations: bool
+    no_text: bool
 
-    def __init__(self, socket: websockets.WebSocketServerProtocol, ctx: Context):
+    def __init__(self, socket: "ServerConnection", ctx: Context) -> None:
         super().__init__(socket)
         self.auth = False
         self.team = None
@@ -175,6 +178,7 @@ class Context:
                       "compatibility": int}
     # team -> slot id -> list of clients authenticated to slot.
     clients: typing.Dict[int, typing.Dict[int, typing.List[Client]]]
+    endpoints: list[Client]
     locations: LocationStore  # typing.Dict[int, typing.Dict[int, typing.Tuple[int, int, int]]]
     location_checks: typing.Dict[typing.Tuple[int, int], typing.Set[int]]
     hints_used: typing.Dict[typing.Tuple[int, int], int]
@@ -364,18 +368,28 @@ async def broadcast_send_encoded_msgs(self, endpoints: typing.Iterable[Endpoint]
             return True
 
     def broadcast_all(self, msgs: typing.List[dict]):
-        msgs = self.dumper(msgs)
-        endpoints = (endpoint for endpoint in self.endpoints if endpoint.auth)
-        async_start(self.broadcast_send_encoded_msgs(endpoints, msgs))
+        msg_is_text = all(msg["cmd"] == "PrintJSON" for msg in msgs)
+        data = self.dumper(msgs)
+        endpoints = (
+            endpoint
+            for endpoint in self.endpoints
+            if endpoint.auth and not (msg_is_text and endpoint.no_text)
+        )
+        async_start(self.broadcast_send_encoded_msgs(endpoints, data))
 
     def broadcast_text_all(self, text: str, additional_arguments: dict = {}):
         self.logger.info("Notice (all): %s" % text)
         self.broadcast_all([{**{"cmd": "PrintJSON", "data": [{ "text": text }]}, **additional_arguments}])
 
     def broadcast_team(self, team: int, msgs: typing.List[dict]):
-        msgs = self.dumper(msgs)
-        endpoints = (endpoint for endpoint in itertools.chain.from_iterable(self.clients[team].values()))
-        async_start(self.broadcast_send_encoded_msgs(endpoints, msgs))
+        msg_is_text = all(msg["cmd"] == "PrintJSON" for msg in msgs)
+        data = self.dumper(msgs)
+        endpoints = (
+            endpoint
+            for endpoint in itertools.chain.from_iterable(self.clients[team].values())
+            if not (msg_is_text and endpoint.no_text)
+        )
+        async_start(self.broadcast_send_encoded_msgs(endpoints, data))
 
     def broadcast(self, endpoints: typing.Iterable[Client], msgs: typing.List[dict]):
         msgs = self.dumper(msgs)
@@ -389,13 +403,13 @@ async def disconnect(self, endpoint: Client):
         await on_client_disconnected(self, endpoint)
 
     def notify_client(self, client: Client, text: str, additional_arguments: dict = {}):
-        if not client.auth:
+        if not client.auth or client.no_text:
             return
         self.logger.info("Notice (Player %s in team %d): %s" % (client.name, client.team + 1, text))
         async_start(self.send_msgs(client, [{"cmd": "PrintJSON", "data": [{ "text": text }], **additional_arguments}]))
 
     def notify_client_multiple(self, client: Client, texts: typing.List[str], additional_arguments: dict = {}):
-        if not client.auth:
+        if not client.auth or client.no_text:
             return
         async_start(self.send_msgs(client,
                                    [{"cmd": "PrintJSON", "data": [{ "text": text }], **additional_arguments}
@@ -760,7 +774,7 @@ def notify_hints(self, team: int, hints: typing.List[Hint], only_new: bool = Fal
             self.on_new_hint(team, slot)
         for slot, hint_data in concerns.items():
             if recipients is None or slot in recipients:
-                clients = self.clients[team].get(slot)
+                clients = filter(lambda c: not c.no_text, self.clients[team].get(slot, []))
                 if not clients:
                     continue
                 client_hints = [datum[1] for datum in sorted(hint_data, key=lambda x: x[0].finding_player != slot)]
@@ -819,7 +833,7 @@ def update_aliases(ctx: Context, team: int):
             async_start(ctx.send_encoded_msgs(client, cmd))
 
 
-async def server(websocket, path: str = "/", ctx: Context = None):
+async def server(websocket: "ServerConnection", path: str = "/", ctx: Context = None) -> None:
     client = Client(websocket, ctx)
     ctx.endpoints.append(client)
 
@@ -910,6 +924,10 @@ async def on_client_joined(ctx: Context, client: Client):
                               "If your client supports it, "
                               "you may have additional local commands you can list with /help.",
                       {"type": "Tutorial"})
+    if not any(isinstance(extension, PerMessageDeflate) for extension in client.socket.extensions):
+        ctx.notify_client(client, "Warning: your client does not support compressed websocket connections! "
+                                  "It may stop working in the future. If you are a player, please report this to the "
+                                  "client's developer.")
     ctx.client_connection_timers[client.team, client.slot] = datetime.datetime.now(datetime.timezone.utc)
 
 
@@ -1803,7 +1821,9 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict):
             ctx.clients[team][slot].append(client)
             client.version = args['version']
             client.tags = args['tags']
-            client.no_locations = 'TextOnly' in client.tags or 'Tracker' in client.tags
+            client.no_locations = "TextOnly" in client.tags or "Tracker" in client.tags
+            # set NoText for old PopTracker clients that predate the tag to save traffic
+            client.no_text = "NoText" in client.tags or ("PopTracker" in client.tags and client.version < (0, 5, 1))
             connected_packet = {
                 "cmd": "Connected",
                 "team": client.team, "slot": client.slot,
@@ -1876,6 +1896,9 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict):
                 client.tags = args["tags"]
                 if set(old_tags) != set(client.tags):
                     client.no_locations = 'TextOnly' in client.tags or 'Tracker' in client.tags
+                    client.no_text = "NoText" in client.tags or (
+                        "PopTracker" in client.tags and client.version < (0, 5, 1)
+                    )
                     ctx.broadcast_text_all(
                         f"{ctx.get_aliased_name(client.team, client.slot)} (Team #{client.team + 1}) has changed tags "
                         f"from {old_tags} to {client.tags}.",

NetUtils.py

@@ -5,7 +5,8 @@
 import warnings
 from json import JSONEncoder, JSONDecoder
 
-import websockets
+if typing.TYPE_CHECKING:
+    from websockets import WebSocketServerProtocol as ServerConnection
 
 from Utils import ByValue, Version
 
@@ -151,7 +152,7 @@ def _object_hook(o: typing.Any) -> typing.Any:
 
 
 class Endpoint:
-    socket: websockets.WebSocketServerProtocol
+    socket: "ServerConnection"
 
     def __init__(self, socket):
         self.socket = socket

Options.py

@@ -1582,7 +1582,7 @@ def dump_player_options(multiworld: MultiWorld) -> None:
             }
             output.append(player_output)
             for option_key, option in world.options_dataclass.type_hints.items():
-                if issubclass(Removed, option):
+                if option.visibility == Visibility.none:
                     continue
                 display_name = getattr(option, "display_name", option_key)
                 player_output[display_name] = getattr(world.options, option_key).current_option_name

docs/network protocol.md

@@ -47,6 +47,9 @@ Packets are simple JSON lists in which any number of ordered network commands ca
 
 An object can contain the "class" key, which will tell the content data type, such as "Version" in the following example.
 
+Websocket connections should support per-message compression. Uncompressed connections are deprecated and may stop
+working in the future.
+
 Example:
 ```javascript
 [{"cmd": "RoomInfo", "version": {"major": 0, "minor": 1, "build": 3, "class": "Version"}, "tags": ["WebHost"], ... }]
@@ -530,9 +533,9 @@ In JSON this may look like:
     {"item": 3, "location": 3, "player": 3, "flags": 0}
 ]
 ```
-`item` is the item id of the item. Item ids are only supported in the range of [-2<sup>53</sup>, 2<sup>53</sup> - 1], with anything ≤ 0 reserved for Archipelago use.
+`item` is the item id of the item. Item ids are only supported in the range of [-2<sup>53</sup> + 1, 2<sup>53</sup> - 1], with anything ≤ 0 reserved for Archipelago use.
 
-`location` is the location id of the item inside the world. Location ids are only supported in the range of [-2<sup>53</sup>, 2<sup>53</sup> - 1], with anything ≤ 0 reserved for Archipelago use.
+`location` is the location id of the item inside the world. Location ids are only supported in the range of [-2<sup>53</sup> + 1, 2<sup>53</sup> - 1], with anything ≤ 0 reserved for Archipelago use.
 
 `player` is the player slot of the world the item is located in, except when inside an [LocationInfo](#LocationInfo) Packet then it will be the slot of the player to receive the item
 
@@ -745,6 +748,7 @@ Tags are represented as a list of strings, the common client tags follow:
 | HintGame  | Indicates the client is a hint game, made to send hints instead of locations. Special join/leave message,¹ `game` is optional.²      |
 | Tracker   | Indicates the client is a tracker, made to track instead of sending locations. Special join/leave message,¹ `game` is optional.²     |
 | TextOnly  | Indicates the client is a basic client, made to chat instead of sending locations. Special join/leave message,¹ `game` is optional.² |
+| NoText    | Indicates the client does not want to receive text messages, improving performance if not needed.                                    |
 
 ¹: When connecting or disconnecting, the chat message shows e.g. "tracking".\
 ²: Allows `game` to be empty or null in [Connect](#connect). Game and version validation will then be skipped.

kvui.py

@@ -26,6 +26,10 @@
 if Utils.is_frozen():
     os.environ["KIVY_DATA_DIR"] = Utils.local_path("data")
 
+import platformdirs
+os.environ["KIVY_HOME"] = os.path.join(platformdirs.user_config_dir("Archipelago", False), "kivy")
+os.makedirs(os.environ["KIVY_HOME"], exist_ok=True)
+
 from kivy.config import Config
 
 Config.set("input", "mouse", "mouse,disable_multitouch")

settings.py

@@ -109,7 +109,7 @@ def changed(self) -> bool:
     def get_type_hints(cls) -> Dict[str, Any]:
         """Returns resolved type hints for the class"""
         if cls._type_cache is None:
-            if not isinstance(next(iter(cls.__annotations__.values())), str):
+            if not cls.__annotations__ or not isinstance(next(iter(cls.__annotations__.values())), str):
                 # non-str: assume already resolved
                 cls._type_cache = cls.__annotations__
             else:
@@ -270,15 +270,20 @@ def dump(self, f: TextIO, level: int = 0) -> None:
             # fetch class to avoid going through getattr
             cls = self.__class__
             type_hints = cls.get_type_hints()
+            entries = [e for e in self]
+            if not entries:
+                # write empty dict for empty Group with no instance values
+                cls._dump_value({}, f, indent="  " * level)
             # validate group
             for name in cls.__annotations__.keys():
                 assert hasattr(cls, name), f"{cls}.{name} is missing a default value"
             # dump ordered members
-            for name in self:
+            for name in entries:
                 attr = cast(object, getattr(self, name))
                 attr_cls = type_hints[name] if name in type_hints else attr.__class__
                 attr_cls_origin = typing.get_origin(attr_cls)
-                while attr_cls_origin is Union:  # resolve to first type for doc string
+                # resolve to first type for doc string
+                while attr_cls_origin is Union or attr_cls_origin is types.UnionType:
                     attr_cls = typing.get_args(attr_cls)[0]
                     attr_cls_origin = typing.get_origin(attr_cls)
                 if attr_cls.__doc__ and attr_cls.__module__ != "builtins":
@@ -787,7 +792,17 @@ def __init__(self, location: Optional[str]):  # change to PathLike[str] once we
         if location:
             from Utils import parse_yaml
             with open(location, encoding="utf-8-sig") as f:
-                options = parse_yaml(f.read())
+                from yaml.error import MarkedYAMLError
+                try:
+                    options = parse_yaml(f.read())
+                except MarkedYAMLError as ex:
+                    if ex.problem_mark:
+                        f.seek(0)
+                        lines = f.readlines()
+                        problem_line = lines[ex.problem_mark.line]
+                        error_line = " " * ex.problem_mark.column + "^"
+                        raise Exception(f"{ex.context} {ex.problem}\n{problem_line}{error_line}")
+                    raise ex
                 # TODO: detect if upgrade is required
                 # TODO: once we have a cache for _world_settings_name_cache, detect if any game section is missing
                 self.update(options or {})

test/general/test_memory.py

@@ -1,5 +1,6 @@
 import unittest
 
+from BaseClasses import MultiWorld
 from worlds.AutoWorld import AutoWorldRegister
 from . import setup_solo_multiworld
 
@@ -9,8 +10,12 @@ def test_leak(self) -> None:
         """Tests that worlds don't leak references to MultiWorld or themselves with default options."""
         import gc
         import weakref
+        refs: dict[str, weakref.ReferenceType[MultiWorld]] = {}
         for game_name, world_type in AutoWorldRegister.world_types.items():
-            with self.subTest("Game", game_name=game_name):
+            with self.subTest("Game creation", game_name=game_name):
                 weak = weakref.ref(setup_solo_multiworld(world_type))
-                gc.collect()
+                refs[game_name] = weak
+        gc.collect()
+        for game_name, weak in refs.items():
+            with self.subTest("Game cleanup", game_name=game_name):
                 self.assertFalse(weak(), "World leaked a reference")

worlds/ahit/Options.py

@@ -338,7 +338,7 @@ class MinExtraYarn(Range):
     There must be at least this much more yarn over the total number of yarn needed to craft all hats.
     For example, if this option's value is 10, and the total yarn needed to craft all hats is 40,
     there must be at least 50 yarn in the pool."""
-    display_name = "Max Extra Yarn"
+    display_name = "Min Extra Yarn"
     range_start = 5
     range_end = 15
     default = 10

worlds/factorio/Options.py

@@ -304,6 +304,11 @@ class EvolutionTrapIncrease(Range):
     range_end = 100
 
 
+class InventorySpillTrapCount(TrapCount):
+    """Trap items that when received trigger dropping your main inventory and trash inventory onto the ground."""
+    display_name = "Inventory Spill Traps"
+
+
 class FactorioWorldGen(OptionDict):
     """World Generation settings. Overview of options at https://wiki.factorio.com/Map_generator,
     with in-depth documentation at https://lua-api.factorio.com/latest/Concepts.html#MapGenSettings"""
@@ -484,6 +489,7 @@ class FactorioOptions(PerGameCommonOptions):
     artillery_traps: ArtilleryTrapCount
     atomic_rocket_traps: AtomicRocketTrapCount
     atomic_cliff_remover_traps: AtomicCliffRemoverTrapCount
+    inventory_spill_traps: InventorySpillTrapCount
     attack_traps: AttackTrapCount
     evolution_traps: EvolutionTrapCount
     evolution_trap_increase: EvolutionTrapIncrease
@@ -518,6 +524,7 @@ class FactorioOptions(PerGameCommonOptions):
             ArtilleryTrapCount,
             AtomicRocketTrapCount,
             AtomicCliffRemoverTrapCount,
+            InventorySpillTrapCount,
         ],
         start_collapsed=True
     ),