Merge pull request #5 from xcoder-tool/new-container-version

danila-schelkov · web-flow · commit 03ad02dac2e1 · 2025-06-25T12:32:10.000+03:00
feat: new container version (6) added
diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 A module for compression like in Supercell games. 
 
-![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/Vorono4ka/sc-compression?include_prereleases)
+![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/xcoder-tool/sc-compression?include_prereleases)
 - 
 
 TestPyPi - https://test.pypi.org/project/sc-compression/ <br>
diff --git a/build.bat b/build.bat
@@ -0,0 +1,6 @@
+@ECHO off
+SET FOLDER="dist"
+DEL /F/Q/S "%FOLDER%" > NUL
+RMDIR /Q/S "%FOLDER%"
+
+python setup.py sdist bdist_wheel
diff --git a/sc_compression/__main__.py b/sc_compression/__main__.py
@@ -80,8 +80,8 @@ def get_extension(file_path: str) -> str:
     return os.path.splitext(file_path)[1][1:]
 
 
-def join_filename(*parts, divider: str = ".") -> str:
-    return divider.join(part for part in parts if type(part) is str and len(part) > 0)
+def join_filename(*parts: str, divider: str = ".") -> str:
+    return divider.join(part for part in parts if len(part) > 0)
 
 
 def process_files(files: list[str], action: Callable[[bytes], bytes], suffix: str):
diff --git a/sc_compression/compressor.py b/sc_compression/compressor.py
@@ -2,7 +2,7 @@
 from hashlib import md5
 
 from sc_compression.signatures import Signatures
-from sc_compression.utils.writer import Writer
+from sc_compression.utils import ByteWriter
 
 try:
     import lzham
@@ -35,7 +35,11 @@ class Compressor:
     ]
 
     def compress(
-        self, data: bytes, signature: Signatures, file_version: int = None
+        self,
+        data: bytes,
+        signature: Signatures,
+        file_version: int = None,
+        metadata: bytes | None = None,
     ) -> bytes:
         uncompressed_size = len(data)
 
@@ -50,7 +54,7 @@ def compress(
         ):
             signature = Signatures.SC
 
-        writer = Writer("little")
+        writer = ByteWriter("little")
         if signature is Signatures.NONE:
             return data
         elif signature in (Signatures.LZMA, Signatures.SIG) or (
@@ -82,27 +86,45 @@ def compress(
         else:
             raise TypeError("Unknown Signature!")
 
-        compressed = self._write_header(compressed, data, file_version, signature)
+        compressed = self._write_header(
+            compressed, data, file_version, signature, metadata
+        )
 
         return compressed
 
     @staticmethod
     def _write_header(
-        compressed: bytes, data: bytes, file_version: int, signature: Signatures
+        compressed: bytes,
+        data: bytes,
+        file_version: int,
+        signature: Signatures,
+        metadata: bytes | None,
     ) -> bytes:
         if signature in (Signatures.SC, Signatures.SCLZ):
-            data_hash = md5(data).digest()
-
-            writer = Writer("big")
+            writer = ByteWriter("big")
             writer.write(b"SC")
-            writer.write_int32(file_version)
-            if file_version == 4:
-                writer.write_int32(1)
-            writer.write_int32(len(data_hash))
-            writer.write(data_hash)
+            if file_version <= 4:
+                writer.write_int32(file_version)
+                if file_version == 4:
+                    writer.write_int32(1)
+
+                data_hash = md5(data).digest()
+
+                writer.write_int32(len(data_hash))
+                writer.write(data_hash)
+            else:
+                writer.set_endian("little")
+                writer.write_int32(file_version)
+
+                if metadata is None:
+                    metadata = b""
+
+                writer.write_int32(len(metadata))
+                writer.write(metadata)
+
             return writer.buffer + compressed
         elif signature == Signatures.SIG:
-            writer = Writer("big")
+            writer = ByteWriter("big")
             writer.write(b"Sig:")
             writer.write(b"\x00" * 64)  # sha64
             return writer.buffer + compressed
diff --git a/sc_compression/decompressor.py b/sc_compression/decompressor.py
@@ -1,8 +1,14 @@
 import lzma
 
 from sc_compression.exceptions import UnknownFileMagicException
-from sc_compression.signatures import Signatures, get_signature, MAGIC_SC, MAGIC_ZSTD, MAGIC_SCLZ
-from sc_compression.utils.reader import Reader
+from sc_compression.signatures import (
+    Signatures,
+    get_signature,
+    MAGIC_SC,
+    MAGIC_ZSTD,
+    MAGIC_SCLZ,
+)
+from sc_compression.utils import ByteReader
 
 try:
     import lzham
@@ -33,38 +39,39 @@ def decompress(self, buffer: bytes) -> bytes:
             return buffer
 
         if self.signature == Signatures.SC:
-            reader = Reader(buffer, "big")
+            reader = ByteReader(buffer, "big")
 
             self.check_magic(MAGIC_SC, reader=reader)
 
             self.file_version = reader.read_int32()
             if self.file_version == 4:
                 self.file_version = reader.read_int32()
 
-            if self.file_version == 0x05000000:
-                reader.endian = "little"
+            if self.file_version == 0x05000000 or self.file_version == 0x06000000:
+                reader.set_endian("little")
+
+                if self.file_version == 0x06000000:
+                    reader.read_int16()
 
                 metadata_table_offset = reader.read_int32()  # offset to metadata vtable
                 reader.read(metadata_table_offset)  # metadata
             else:
                 hash_length = reader.read_int32()
                 self.hash = reader.read(hash_length)
 
-            return self.decompress(buffer[reader.tell() :])
+            return self.decompress(reader.read_all_bytes())
         elif self.signature == Signatures.SIG:
             return self.decompress(buffer[68:])
         elif self.signature == Signatures.SCLZ and lzham is not None:
-            reader = Reader(buffer, "little")
+            reader = ByteReader(buffer, "little")
 
             self.check_magic(MAGIC_SCLZ, reader=reader)
 
             dict_size_log2 = reader.read_u_int8()
             uncompressed_size = reader.read_int32()
 
             filters = {"dict_size_log2": dict_size_log2}
-            return lzham.decompress(
-                reader.buffer[reader.tell() :], uncompressed_size, filters
-            )
+            return lzham.decompress(reader.read_all_bytes(), uncompressed_size, filters)
         elif self.signature == Signatures.LZMA:
             decompressor = lzma.LZMADecompressor()
             compressed = buffer[:5] + b"\xff" * 8 + buffer[9:]
@@ -78,11 +85,15 @@ def decompress(self, buffer: bytes) -> bytes:
             raise TypeError(self.signature)
 
     @staticmethod
-    def check_magic(expected_magic: bytes, reader: Reader | None = None, buffer: bytes | None = None) -> None:
+    def check_magic(
+        expected_magic: bytes,
+        reader: ByteReader | None = None,
+        buffer: bytes | None = None,
+    ) -> None:
         if reader is not None:
             magic = reader.read(len(expected_magic))
         elif buffer is not None:
-            magic = buffer[:len(expected_magic)]
+            magic = buffer[: len(expected_magic)]
         else:
             raise Exception("Cannot find buffer to get magic from.")
 
diff --git a/sc_compression/exceptions/unknown_file_magic.py b/sc_compression/exceptions/unknown_file_magic.py
@@ -1,3 +1,6 @@
 class UnknownFileMagicException(Exception):
     def __init__(self, magic: bytes, expected_magic: bytes, *args):
-        super().__init__(f"Unknown file magic: {magic.hex()}, while expected was {expected_magic.hex()}", *args)
+        super().__init__(
+            f"Unknown file magic: {magic.hex()}, while expected was {expected_magic.hex()}",
+            *args,
+        )
diff --git a/sc_compression/signatures.py b/sc_compression/signatures.py
@@ -1,13 +1,15 @@
+import enum
 import re
-from enum import Enum
+from enum import IntEnum
 
-MAGIC_SC = b"SC"
-MAGIC_SCLZ = b"SCLZ"
-MAGIC_SIG = b"Sig:"
-MAGIC_ZSTD = b"\x28\xb5\x2f\xfd"
+MAGIC_SC: bytes = b"SC"
+MAGIC_SCLZ: bytes = b"SCLZ"
+MAGIC_SIG: bytes = b"Sig:"
+MAGIC_ZSTD: bytes = b"\x28\xb5\x2f\xfd"
 
 
-class Signatures(Enum):
+@enum.unique
+class Signatures(IntEnum):
     NONE = 0
     LZMA = 1
     SC = 2
diff --git a/sc_compression/support/lzham.py b/sc_compression/support/lzham.py
@@ -20,7 +20,7 @@ def decompress(data, uncompressed_size, filters):
         decompressed_path = mktemp(".lzham")
         if system(
             f"{path.dirname(__file__)}/lzham.exe "
-            f'-c -d{filters["dict_size_log2"]} '
+            f"-c -d{filters['dict_size_log2']} "
             f"d {temp_file_path} {decompressed_path} > nul 2>&1"
         ):
             remove(temp_file_path)
@@ -43,7 +43,7 @@ def compress(data, filters):
         compressed_path = mktemp(".lzham")
         if system(
             f"{path.dirname(__file__)}/lzham.exe "
-            f'-c -d{filters["dict_size_log2"]} '
+            f"-c -d{filters['dict_size_log2']} "
             f"c {temp_file_path} {compressed_path} > nul 2>&1"
         ):
             remove(temp_file_path)
diff --git a/sc_compression/utils/__init__.py b/sc_compression/utils/__init__.py
@@ -1 +1,4 @@
-__all__ = ["reader", "writer"]
+from .reader import ByteReader
+from .writer import ByteWriter
+
+__all__ = ["ByteWriter", "ByteReader"]
diff --git a/sc_compression/utils/common.py b/sc_compression/utils/common.py
@@ -0,0 +1,14 @@
+from typing import Literal
+
+from typing_extensions import TypeAlias
+
+Endian: TypeAlias = Literal["big", "little"]
+
+
+def get_endian_sign(endian: Literal["big", "little"]) -> Literal[">", "<"]:
+    if endian == "big":
+        return ">"
+    elif endian == "little":
+        return "<"
+
+    raise NotImplementedError(f"Unknown endian requested: {endian}")
diff --git a/sc_compression/utils/reader.py b/sc_compression/utils/reader.py
@@ -1,89 +1,50 @@
-import struct
-from typing import Literal
+from io import BufferedReader, BytesIO
+from struct import unpack
 
+from .common import get_endian_sign, Endian
 
-ENDIAN_SIGN = {"big": ">", "little": "<"}
 
+class ByteReader:
+    def __init__(self, initial_bytes: bytes, endian: Endian):
+        # noinspection PyTypeChecker
+        self._internal_reader = BufferedReader(BytesIO(initial_bytes))
+        self._endian_sign = get_endian_sign(endian)
 
-class Reader:
-    def __init__(self, buffer: bytes, endian: Literal["little", "big"] = "big"):
-        self.buffer = buffer
-        self.endian: Literal["little", "big"] = endian
-        self._position = 0
+    def set_endian(self, endian: Endian) -> None:
+        self._endian_sign = get_endian_sign(endian)
 
-    def read(self, length: int = 1):
-        result = self.buffer[self._position : self._position + length]
-        self._position += length
+    def seek(self, position: int) -> None:
+        self._internal_reader.seek(position)
 
-        return result
+    def tell(self) -> int:
+        return self._internal_reader.tell()
 
-    def read_u_integer(self, length: int = 1) -> int:
-        return int.from_bytes(self.read(length), self.endian, signed=False)
+    def read(self, size: int) -> bytes:
+        return self._internal_reader.read(size)
 
-    def read_integer(self, length: int = 1) -> int:
-        return int.from_bytes(self.read(length), self.endian, signed=True)
+    def read_all_bytes(self) -> bytes:
+        return self._internal_reader.read()
 
     def read_u_int64(self) -> int:
-        return self.read_u_integer(8)
+        return unpack(f"{self._endian_sign}Q", self.read(8))[0]
 
     def read_int64(self) -> int:
-        return self.read_integer(8)
-
-    def read_float(self) -> float:
-        (floating,) = struct.unpack(ENDIAN_SIGN[self.endian] + "f", self.read(4))
-        return floating
+        return unpack(f"{self._endian_sign}q", self.read(8))[0]
 
     def read_u_int32(self) -> int:
-        return self.read_u_integer(4)
+        return unpack(f"{self._endian_sign}I", self.read(4))[0]
 
     def read_int32(self) -> int:
-        return self.read_integer(4)
-
-    def read_nu_int16(self) -> float:
-        return self.read_u_int16() / 65535
+        return unpack(f"{self._endian_sign}i", self.read(4))[0]
 
     def read_u_int16(self) -> int:
-        return self.read_u_integer(2)
-
-    def read_n_int16(self) -> float:
-        return self.read_int16() / 32512
+        return unpack(f"{self._endian_sign}H", self.read(2))[0]
 
     def read_int16(self) -> int:
-        return self.read_integer(2)
+        return unpack(f"{self._endian_sign}h", self.read(2))[0]
 
     def read_u_int8(self) -> int:
-        return self.read_u_integer()
+        return unpack(f"{self._endian_sign}B", self.read(1))[0]
 
     def read_int8(self) -> int:
-        return self.read_integer()
-
-    def read_bool(self) -> bool:
-        if self.read_u_int8() >= 1:
-            return True
-        else:
-            return False
-
-    readUInt = read_u_integer
-    readInt = read_integer
-
-    readULong = read_u_int64
-    readLong = read_int64
-
-    readNUShort = read_nu_int16
-    readNShort = read_n_int16
-
-    readUShort = read_u_int16
-    readShort = read_int16
-
-    readUByte = read_u_int8
-    readByte = read_int8
-
-    def read_char(self, length: int = 1) -> str:
-        return self.read(length).decode("utf-8")
-
-    def read_string(self) -> str:
-        length = self.readUShort()
-        return self.read_char(length)
-
-    def tell(self) -> int:
-        return self._position
+        return unpack(f"{self._endian_sign}b", self.read(1))[0]
diff --git a/sc_compression/utils/writer.py b/sc_compression/utils/writer.py
diff --git a/setup.py b/setup.py
diff --git a/test_upload.cmd b/test_upload.cmd
diff --git a/upload.cmd b/upload.cmd

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`
`3`	`3`	`A module for compression like in Supercell games.`
`4`	`4`
`5`		`-![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/Vorono4ka/sc-compression?include_prereleases)`
	`5`	`+![GitHub release (latest by date including pre-releases)](https://img.shields.io/github/v/release/xcoder-tool/sc-compression?include_prereleases)`
`6`	`6`	`-`
`7`	`7`
`8`	`8`	`TestPyPi - https://test.pypi.org/project/sc-compression/ <br>`