From 5885aec96ce8ed6bd3cff0a696b6d8c4e404d7bc Mon Sep 17 00:00:00 2001 From: corepunch Date: Sun, 4 Oct 2026 09:56:36 +0200 Subject: [PATCH] Fix missing music decoder in Linux releases --- .github/docker/ci/Dockerfile | 6 ++ .github/workflows/c-cpp.yml | 24 +++++++ .github/workflows/release.yml | 12 +++- README.md | 14 ++++ dist-scripts/linux/bundle_media.py | 53 ++++++++++++++ docs/games/warcraft-3/music.md | 17 ++++- games/warcraft-3/game.mk | 6 +- games/warcraft-3/tests/test_commands.c | 24 +++++++ tests/test_linux_media_release.py | 98 ++++++++++++++++++++++++++ 9 files changed, 251 insertions(+), 3 deletions(-) create mode 100644 dist-scripts/linux/bundle_media.py create mode 100644 tests/test_linux_media_release.py diff --git a/.github/docker/ci/Dockerfile b/.github/docker/ci/Dockerfile index 3ae7a9ffd..8be1ad8a0 100644 --- a/.github/docker/ci/Dockerfile +++ b/.github/docker/ci/Dockerfile @@ -9,10 +9,16 @@ RUN apt-get update \ build-essential \ ca-certificates \ git \ + libavcodec-dev \ + libavformat-dev \ + libavutil-dev \ libgles2-mesa-dev \ libgl1-mesa-dri \ libjpeg-dev \ libsdl2-dev \ + libswresample-dev \ + libswscale-dev \ + patchelf \ pkg-config \ python3 \ zlib1g-dev \ diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml index 0450f9ef8..2cbc1aa95 100644 --- a/.github/workflows/c-cpp.yml +++ b/.github/workflows/c-cpp.yml @@ -8,6 +8,30 @@ on: branches: ["main"] jobs: + media: + name: Linux release music and runtime packaging + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - name: Build Linux dependency environment + run: docker build --tag openrealm-media-ci .github/docker/ci + - name: Build and verify the release media package + run: | + docker run --rm --mount "type=bind,source=$PWD,target=/workspace" \ + --workdir /workspace openrealm-media-ci bash -ec ' + make BUILD=release FFMPEG=1 BIN_DIR=build/media/bin LIB_DIR=build/media/lib SHARE_INSTALL=build/media/share openwarcraft3 + make BUILD=release BIN_DIR=build/media/bin LIB_DIR=build/media/lib test-sound + mkdir -p build/media/package/bin build/media/package/lib + cp build/media/bin/openwarcraft3 build/media/package/bin/ + cp build/media/lib/*.so build/media/package/lib/ + python3 dist-scripts/linux/bundle_media.py build/media/bin/openwarcraft3 build/media/package/lib + ldd build/media/package/bin/openwarcraft3 > build/media/package/dependencies.txt + cat build/media/package/dependencies.txt + ! grep -q "not found" build/media/package/dependencies.txt + ' + eos: name: EOS regressions and live multiplayer checks # Fork PRs cannot read the licensed SDK/player credentials. Run their normal diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a31484cee..3ebaea724 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -55,10 +55,16 @@ jobs: build-essential \ ca-certificates \ git \ + libavcodec-dev \ + libavformat-dev \ + libavutil-dev \ libgl-dev \ libgles2-mesa-dev \ libjpeg-dev \ libsdl2-dev \ + libswresample-dev \ + libswscale-dev \ + patchelf \ pkg-config \ python3 \ zlib1g-dev @@ -99,7 +105,8 @@ jobs: - name: Build (Linux) if: runner.os == 'Linux' - run: make BUILD=release EOS=1 EOS_CONFIG_FILE=data/eos/eos.cfg build + # FFmpeg also decodes background music; omitting it made releases silent. + run: make BUILD=release FFMPEG=1 EOS=1 EOS_CONFIG_FILE=data/eos/eos.cfg build - name: Build (macOS) if: runner.os == 'macOS' @@ -134,6 +141,9 @@ jobs: cp -r build/lib/. release/lib/ cp -r build/share release/ cp data/eos/ThirdPartyNotices/ThirdPartySoftwareNotice.txt release/licenses/EOS/ + if [ "${{ runner.os }}" = "Linux" ]; then + python3 dist-scripts/linux/bundle_media.py build/bin/openwarcraft3 release/lib + fi tar -czf ${{ matrix.archive }} -C release . - name: Create release archive (Windows) diff --git a/README.md b/README.md index 1e242c340..6435ae118 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,20 @@ cd open-realm make build ``` +To include Warcraft III video playback and background music, install the FFmpeg development libraries and `pkg-config`, then build with: + +```bash +# Ubuntu/Debian +sudo apt-get install pkg-config libavformat-dev libavcodec-dev libavutil-dev libswscale-dev libswresample-dev +# Manjaro/Arch +sudo pacman -S --needed pkgconf ffmpeg + +make clean +make build -j4 FFMPEG=1 +``` + +On macOS, install these dependencies with `brew install pkg-config ffmpeg`. Ordinary sound effects and unit voices work without `FFMPEG=1`. See [Pre-Rendered Movies](docs/games/warcraft-3/pre-rendered-movies.md). + Useful targets: ```bash diff --git a/dist-scripts/linux/bundle_media.py b/dist-scripts/linux/bundle_media.py new file mode 100644 index 000000000..44034fa91 --- /dev/null +++ b/dist-scripts/linux/bundle_media.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Bundle the Linux FFmpeg runtime closure without replacing the host's libc.""" + +import argparse +from pathlib import Path +import re +import shutil +import subprocess + +REQUIRED = ("libavformat", "libavcodec", "libavutil", "libswscale", "libswresample") +DOC_ROOT = Path("/usr/share/doc") +HOST_RUNTIME = re.compile(r"(?:libc|libm|libdl|libpthread|librt|libresolv|libgcc_s|libstdc\+\+)\.so\.|ld-linux|lib(?:drm|va[.-]|vdpau|GL|EGL|vulkan|OpenCL|X|xcb)") + + +def dependencies(paths): + result = subprocess.run(["ldd", *map(str, paths)], check=True, capture_output=True, text=True) + if "not found" in result.stdout: + raise RuntimeError("Unresolved media dependencies:\n" + result.stdout) + return {Path(match.group(1)) for match in re.finditer(r"=> (/[^\n]+?) \(", result.stdout)} + + +def bundle(binary, destination): + linked = dependencies([binary]) + roots = [path for path in linked if any(path.name.startswith(name + ".so.") for name in REQUIRED)] + if not all(any(path.name.startswith(name + ".so.") for path in roots) for name in REQUIRED): + raise RuntimeError("Linux releases require FFMPEG=1 for background music playback") + libraries = sorted(path for path in set(roots) | dependencies(roots) if not HOST_RUNTIME.match(path.name)) + notices = {} + for library in libraries: + owner = subprocess.run(["dpkg-query", "-S", str(library.resolve())], check=True, capture_output=True, text=True) + package = owner.stdout.split(": ", 1)[0].split(":", 1)[0] + notice = DOC_ROOT / package / "copyright" + if not notice.is_file(): + raise RuntimeError(f"Missing copyright notice for {library}: {notice}") + notices[package] = notice + destination.mkdir(parents=True, exist_ok=True) + for library in libraries: + output = destination / library.name + shutil.copyfile(library, output) + # Executable RUNPATH is not inherited for FFmpeg's transitive libraries. + subprocess.run(["patchelf", "--set-rpath", "$ORIGIN", str(output)], check=True) + for package, notice in notices.items(): + output = destination.parent / "licenses/ffmpeg" / package / "copyright" + output.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(notice, output) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("destination", type=Path) + args = parser.parse_args() + bundle(args.binary, args.destination) diff --git a/docs/games/warcraft-3/music.md b/docs/games/warcraft-3/music.md index 236d158a2..674dbe78d 100644 --- a/docs/games/warcraft-3/music.md +++ b/docs/games/warcraft-3/music.md @@ -26,10 +26,25 @@ Do not route background music through `svc_sound`, `S_PlaySoundFile`, unit chann The default build keeps FFmpeg optional. Music commands and state still exist without FFmpeg, but no compressed music decoder is available and playback remains silent. +The Linux GitHub release must explicitly build with `FFMPEG=1`. The v0.0.10-alpha release workflow invoked +`make BUILD=release EOS=1 ... build` without that flag, so its `CL_MusicOpenTrack` was the stub that always returns false. +This disables menu and map music independently of SDL audio initialization. A local `FFMPEG=1` build therefore has music +where that release is silent; it does not demonstrate an SDL or ordinary WAV/MP3 effect decoder failure. + +`dist-scripts/linux/bundle_media.py` verifies that all five media libraries are linked and copies their complete runtime +dependency closure into `release/lib`, setting each copy's RUNPATH to `$ORIGIN`. The executable's RUNPATH alone does not +resolve transitive dependencies. Keep glibc, C++/GCC runtimes and graphics driver interfaces on the host so Ubuntu copies +cannot replace the libraries required by Manjaro's newer Mesa/SDL drivers. Retain Debian package copyright notices under +`release/licenses/ffmpeg`, and fail packaging on unresolved dependencies or missing decoder libraries. This avoids requiring +Manjaro's current FFmpeg to provide Ubuntu's older library SONAMEs. The `Linux release music and runtime packaging` CI job +builds the optimized Linux decoder and verifies the packaged dependency closure; `make test-linux-media-release` covers +missing decoders, missing dependencies, RUNPATHs, notices, and paths containing spaces and UTF-8 characters. + Enable the decoder with: ```bash -make FFMPEG=1 +make clean +make build -j4 FFMPEG=1 ``` The Warcraft III build already uses these pkg-config libraries for pre-rendered movies and now reuses them for music: diff --git a/games/warcraft-3/game.mk b/games/warcraft-3/game.mk index 0de47e52f..de7f9a520 100644 --- a/games/warcraft-3/game.mk +++ b/games/warcraft-3/game.mk @@ -199,6 +199,10 @@ test-eos-release: python3 tests/test_eos_acceptance_runner.py python3 tests/test_eos_release.py +.PHONY: test-linux-media-release +test-linux-media-release: + python3 tests/test_linux_media_release.py + # Golden-image render regression test (deterministic MDX renders vs committed # references). Requires a display/GL, so it is opt-in and NOT part of `make test` # (CI is headless). Run locally after renderer changes. @@ -294,7 +298,7 @@ TEST_UI_SRCS := \ TEST_JOBS ?= 16 - test: test-eos-release test-menu-boundary test-assets $(SHARED_LIB) $(JASS_LIB) $(SHEET_LIB) | $(BIN_DIR) $(TEST_JUNIT_DIR) + test: test-eos-release test-linux-media-release test-menu-boundary test-assets $(SHARED_LIB) $(JASS_LIB) $(SHEET_LIB) | $(BIN_DIR) $(TEST_JUNIT_DIR) @rm -f $(TEST_JUNIT_DIR)/*.xml @$(CC) $(TEST_CFLAGS) -DBZ_TESTS -o $(BIN_DIR)/test_openwarcraft3$(EXE_EXT) \ tests/test_runner.c tests/test_compat.c tests/test_net.c tests/test_online_packet.c tests/test_tool_common.c \ diff --git a/games/warcraft-3/tests/test_commands.c b/games/warcraft-3/tests/test_commands.c index 796be2973..c7163009a 100644 --- a/games/warcraft-3/tests/test_commands.c +++ b/games/warcraft-3/tests/test_commands.c @@ -186,6 +186,30 @@ TEST(commands, save_path_adds_one_sav_extension) { T_STREQ(path, "/tmp/openwarcraft3-save-path-test/saves/manual.SAV"); } +/* Linux filenames preserve UTF-8 bytes; a quoted data directory may also contain spaces. */ +TEST(commands, media_paths_preserve_utf8_and_spaces) { + PATHSTR old_home, path, resolved; + cstring_t root = "build/tests/Téléchargements/Games WIP/ROC"; + uint32_t size = 0; + setup_command_tests(); + snprintf(old_home, sizeof(old_home), "%s", FS_HomePath()); + FS_SetHomeDirectory(root); + FS_UserPath("audio-path-test.wav", path, sizeof(path)); + FILE *file = fopen(path, "wb"); + T_NOT_NULL(file); + if (file) { T_EQ(fwrite("test", 1, 4, file), 4); fclose(file); } + T_ASSERT(FS_AddDataDirectory(root)); + T_ASSERT(FS_ResolveLoosePath("audio-path-test.wav", resolved, sizeof(resolved))); + T_STREQ(resolved, path); + char *data = FS_ReadFile("audio-path-test.wav", &size); + T_NOT_NULL(data); + T_EQ(size, 4); + if (data) T_ASSERT(!memcmp(data, "test", 4)); + FS_FreeFile(data); + remove(path); + FS_SetHomeDirectory(old_home); +} + TEST(commands, save_list_returns_newest_sav_basenames_first) { PATHSTR older, newer, ignored; char list[256] = { 0 }; diff --git a/tests/test_linux_media_release.py b/tests/test_linux_media_release.py new file mode 100644 index 000000000..8647dade2 --- /dev/null +++ b/tests/test_linux_media_release.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Linux media packaging regressions; no FFmpeg or Linux host required.""" + +import importlib.util +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("linux_media", ROOT / "dist-scripts/linux/bundle_media.py") +media = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(media) + + +class LinuxMediaReleaseTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.library = self.root / "Téléchargements/Games WIP/system" + self.library.mkdir(parents=True) + self.binary = self.root / "openwarcraft3" + self.binary.write_bytes(b"executable fixture") + self.destination = self.root / "release/lib" + self.docs = self.root / "doc" + self.commands = [] + + def library_file(self, name): + path = self.library / name + path.write_bytes(name.encode()) + package = "fixture-" + name.split(".")[0] + notice = self.docs / package / "copyright" + notice.parent.mkdir(parents=True, exist_ok=True) + notice.write_text(package + " license") + return path + + def run_tool(self, command, **kwargs): + self.commands.append(command) + if command[0] == "ldd": + paths = self.roots if command[1:] == [str(self.binary)] else self.closure + output = "".join(f"{p.name} => {p} (0x1234)\n" for p in paths) + elif command[0] == "dpkg-query": + output = "fixture-" + Path(command[-1]).name.split(".")[0] + ":amd64: " + command[-1] + else: + output = "" + return subprocess.CompletedProcess(command, 0, output, "") + + def test_bundles_media_closure_with_local_rpaths_and_licenses(self): + self.roots = [self.library_file(name + ".so.60") for name in media.REQUIRED] + codec = self.library_file("libcodec-support.so.1") + libc = self.library_file("libc.so.6") + driver = self.library_file("libdrm.so.2") + cpp = self.library_file("libstdc++.so.6") + self.closure = self.roots + [codec, libc, driver, cpp] + with patch.object(media, "DOC_ROOT", self.docs), patch.object(media.subprocess, "run", self.run_tool): + media.bundle(self.binary, self.destination) + for path in self.roots + [codec]: + self.assertEqual((self.destination / path.name).read_bytes(), path.read_bytes()) + self.assertIn(["patchelf", "--set-rpath", "$ORIGIN", str(self.destination / path.name)], self.commands) + self.assertFalse((self.destination / libc.name).exists()) + self.assertFalse((self.destination / driver.name).exists()) + self.assertFalse((self.destination / cpp.name).exists()) + self.assertEqual(len(list((self.destination.parent / "licenses/ffmpeg").glob("*/copyright"))), 6) + + def test_disabled_decoder_fails_before_packaging(self): + self.roots = [] + with patch.object(media.subprocess, "run", self.run_tool): + with self.assertRaisesRegex(RuntimeError, "FFMPEG=1"): + media.bundle(self.binary, self.destination) + self.assertFalse(self.destination.exists()) + + def test_unresolved_runtime_fails_before_packaging(self): + result = subprocess.CompletedProcess([], 0, "libavcodec.so.60 => not found\n", "") + with patch.object(media.subprocess, "run", return_value=result): + with self.assertRaisesRegex(RuntimeError, "not found"): + media.bundle(self.binary, self.destination) + self.assertFalse(self.destination.exists()) + + def test_linux_release_enables_music_decoder(self): + workflow = (ROOT / ".github/workflows/release.yml").read_text() + linux = workflow.split("- name: Build (Linux)\n", 1)[1].split("- name:", 1)[0] + self.assertIn("FFMPEG=1", linux) + + def test_missing_copyright_notice_fails_before_packaging(self): + self.roots = [self.library_file(name + ".so.60") for name in media.REQUIRED] + self.closure = self.roots + for notice in self.docs.glob("*/copyright"): + notice.unlink() + with patch.object(media, "DOC_ROOT", self.docs), patch.object(media.subprocess, "run", self.run_tool): + with self.assertRaisesRegex(RuntimeError, "Missing copyright"): + media.bundle(self.binary, self.destination) + self.assertFalse(self.destination.exists()) + + +if __name__ == "__main__": + unittest.main()