From f2f495e2bbcdd925b34971fc74da8bd09be3ad79 Mon Sep 17 00:00:00 2001 From: Yanick Fratantonio Date: Wed, 29 Jan 2025 15:09:46 +0000 Subject: [PATCH 1/6] python: add placeholder for magika for for the pure python wheel "magika" will point to this script, which informs the user that the rust client is not there, and it provides pointers on how to report issues and use the python client. --- .../magika_rust_client_not_found_warning.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100755 python/src/magika/cli/magika_rust_client_not_found_warning.py diff --git a/python/src/magika/cli/magika_rust_client_not_found_warning.py b/python/src/magika/cli/magika_rust_client_not_found_warning.py new file mode 100755 index 00000000..f80373b5 --- /dev/null +++ b/python/src/magika/cli/magika_rust_client_not_found_warning.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + + +def main() -> None: + message = """ +WARNING: you have attempted to run `$ magika` (the Rust client), but this is not +available in the python package you installed, likely because magika pipeline +does not currently build binary wheels compatible with your platform settings. + +If you think this is a problem worth solving, please open an issue at +https://github.com/google/magika. + +In the meantime, you can use the old python magika client with `$ magika-python-client`. +""" + + print(message.strip()) + sys.exit(1) + + +if __name__ == "__main__": + main() From 9c33ada74af9a02b80fc4e7f8979a723aee84d04 Mon Sep 17 00:00:00 2001 From: Yanick Fratantonio Date: Wed, 29 Jan 2025 15:10:12 +0000 Subject: [PATCH 2/6] python: expose the python client as "magika-python-client" --- python/scripts/prepare_pyproject_for_pure_python_wheel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/scripts/prepare_pyproject_for_pure_python_wheel.py b/python/scripts/prepare_pyproject_for_pure_python_wheel.py index 6b7d9c95..4be3375a 100644 --- a/python/scripts/prepare_pyproject_for_pure_python_wheel.py +++ b/python/scripts/prepare_pyproject_for_pure_python_wheel.py @@ -35,7 +35,8 @@ def main() -> None: # Make the python's magika client available as a script pyproject_content["project"]["scripts"] = { - "magika": "magika.cli.magika_client:main", + "magika-python-client": "magika.cli.magika_client:main", + "magika": "magika.cli.magika_rust_client_not_found_warning:main", } pyproject_toml_path.write_text(tomli_w.dumps(pyproject_content)) From 456b9f9bf3a1745c431a8631aa10050443591dd8 Mon Sep 17 00:00:00 2001 From: Yanick Fratantonio Date: Wed, 29 Jan 2025 15:10:31 +0000 Subject: [PATCH 3/6] gh workflow: adapt the python build workflow for the new pure python wheel --- .github/workflows/python-build-package.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-build-package.yml b/.github/workflows/python-build-package.yml index 0d209ea1..ef0ceebb 100644 --- a/.github/workflows/python-build-package.yml +++ b/.github/workflows/python-build-package.yml @@ -224,8 +224,13 @@ jobs: uv add ".\$wheel" - name: Install the wheel run: python3 -m pip install $(python -c "import glob; print(glob.glob('dist/*.whl')[0])") - - run: magika --version + + # Check that the magika script points to the placeholder raising a warning + - run: magika --version | grep -C10 WARNING | grep -C10 magika-python-client + # Check that the fallback magika's python client can be run + - run: magika-python-client -r tests_data/basic + # Check that the results of the python's client are correct + - run: python3 ./python/scripts/run_quick_test_magika_cli.py --client-path magika-python-client + # Test the python module - run: "python3 -c 'import magika; m = magika.Magika(); print(m)'" - - run: magika -r tests_data/basic - - run: python3 ./python/scripts/run_quick_test_magika_cli.py - run: python3 ./python/scripts/run_quick_test_magika_module.py From 625d460fb3daf4fd14e1ad6e6f7895b6c217bd0c Mon Sep 17 00:00:00 2001 From: Yanick Fratantonio Date: Wed, 29 Jan 2025 15:23:12 +0000 Subject: [PATCH 4/6] gh workflow: simplify python's build workflow --- .github/workflows/python-build-package.yml | 40 ---------------------- 1 file changed, 40 deletions(-) diff --git a/.github/workflows/python-build-package.yml b/.github/workflows/python-build-package.yml index ef0ceebb..9c94e96d 100644 --- a/.github/workflows/python-build-package.yml +++ b/.github/workflows/python-build-package.yml @@ -57,32 +57,6 @@ jobs: target: ${{ matrix.platform.target }} args: --release --out=../dist working-directory: python - - if: matrix.platform.runner != 'windows-latest' - name: Check that `uv add magika.whl` works - run: | - mkdir /tmp/test-uv - cp -vR dist/*.whl /tmp/test-uv - cd /tmp/test-uv - uv init - uv add ./$(\ls -1 *.whl | head -n 1) - - if: matrix.platform.runner == 'windows-latest' - name: Check that `uv add magika.whl` works - shell: pwsh - run: | - mkdir C:\test-uv - Copy-Item -Path dist\*.whl -Destination C:\test-uv - cd C:\test-uv - $env:PATH += ";$HOME/.local/bin" - uv init - $wheel = Get-ChildItem -Filter *.whl | Select-Object -ExpandProperty Name - uv add ".\$wheel" - - name: Install wheels - run: python3 -m pip install $(python -c "import glob; print(glob.glob('dist/*.whl')[0])") - - run: magika --version - - run: "python3 -c 'import magika; m = magika.Magika(); print(m)'" - - run: magika -r tests_data/basic - - run: python3 ./python/scripts/run_quick_test_magika_cli.py - - run: python3 ./python/scripts/run_quick_test_magika_module.py - name: Upload wheels uses: actions/upload-artifact@v4 with: @@ -155,20 +129,6 @@ jobs: - name: Build pure python wheel run: uv build --wheel --out-dir ../dist working-directory: python - - name: Check that `uv add magika.whl` works - run: | - mkdir /tmp/test-uv - cp -vR dist/*.whl /tmp/test-uv - cd /tmp/test-uv - uv init - uv add ./$(\ls -1 *.whl | head -n 1) - - name: Install the wheel - run: python3 -m pip install $(python -c "import glob; print(glob.glob('dist/*.whl')[0])") - - run: magika --version - - run: "python3 -c 'import magika; m = magika.Magika(); print(m)'" - - run: magika -r tests_data/basic - - run: python3 ./python/scripts/run_quick_test_magika_cli.py - - run: python3 ./python/scripts/run_quick_test_magika_module.py - name: Upload wheel uses: actions/upload-artifact@v4 with: From 91adadd9a2cce532373cb2c7911de777243e6fc9 Mon Sep 17 00:00:00 2001 From: Yanick Fratantonio Date: Wed, 29 Jan 2025 15:30:42 +0000 Subject: [PATCH 5/6] python: cli tester should not resolve the path or assume existance --- python/scripts/run_quick_test_magika_cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/scripts/run_quick_test_magika_cli.py b/python/scripts/run_quick_test_magika_cli.py index 0e29d868..815adfa6 100755 --- a/python/scripts/run_quick_test_magika_cli.py +++ b/python/scripts/run_quick_test_magika_cli.py @@ -34,12 +34,13 @@ @click.command() @click.option( "--client-path", - type=click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True), + type=click.Path(exists=False, resolve_path=False), ) def main(client_path: Optional[Path]) -> None: """Tests the Rust or Python Magika client. By default, it runs "magika" (expected in PATH). Use --client-path to specify a different client - executable. + executable. Note that "client_path" may not point to a valid file, but still + be a valid target to run as it may be found in the PATH. """ basic_tests_dir = ( From 01a156fc3c912f3f3560aa6bfbae4be843bd492d Mon Sep 17 00:00:00 2001 From: Yanick Fratantonio Date: Wed, 29 Jan 2025 15:49:03 +0000 Subject: [PATCH 6/6] gh workflow: generate sdist alongside pure python wheel Fixes #919. --- .github/workflows/python-build-package.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-build-package.yml b/.github/workflows/python-build-package.yml index 9c94e96d..60d5f587 100644 --- a/.github/workflows/python-build-package.yml +++ b/.github/workflows/python-build-package.yml @@ -118,7 +118,7 @@ jobs: - run: python3 ./python/scripts/run_quick_test_magika_cli.py - run: python3 ./python/scripts/run_quick_test_magika_module.py - build-pure-python-wheel: + build-pure-python-wheel-and-sdist: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -126,18 +126,18 @@ jobs: run: curl -LsSf https://astral.sh/uv/0.5.22/install.sh | sh - run: uv run ./scripts/prepare_pyproject_for_pure_python_wheel.py working-directory: python - - name: Build pure python wheel - run: uv build --wheel --out-dir ../dist + - name: Build pure python wheel and source distribution + run: uv build --out-dir ../dist working-directory: python - - name: Upload wheel + - name: Upload wheel and sdist uses: actions/upload-artifact@v4 with: - name: wheel-pure-python + name: wheel-pure-python-and-sdist path: dist # Download, install, and test the pure python wheel on multiple platforms test-pure-python-wheel: - needs: [build-pure-python-wheel] + needs: [build-pure-python-wheel-and-sdist] runs-on: ${{ matrix.platform.runner }} strategy: # We want to know in which exact situation the tests fail @@ -155,7 +155,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: - name: wheel-pure-python + name: wheel-pure-python-and-sdist path: dist - name: Setup Python uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # pin@v5