From c603b4f163e87425ffa49538c33137a12eb13852 Mon Sep 17 00:00:00 2001 From: Basile Dura Date: Fri, 28 Apr 2023 11:31:51 +0200 Subject: [PATCH] feat: add release utilities (#14) * feat: port git information on setup from spaCy add corresponding numpy licence remove auto-generated git_info submodule * feat: add tagging helper scripts * feat: update about.py to include __title__ key * feat: add a version key in setup.cfg * feat: add automatic version extraction from cfg file * fix: adapt release utilities to new _version file * feat: get package name from setup.cfg * build: add write_version_py to the setup procedure * fix: version extraction in setup.py * fix: underline in licenses/3rd_party_licenses.txt Co-authored-by: Sofie Van Landeghem * build: remove git_info at build time * chore: remove irrelevant 3rd party license * fix: adapt code to check spaCy git commit * fix: use weasel version and drop use of git commit for lockfile * chore: change first version identifier * feat: add check for deprecated spaCy env vars * test: env var check * fix: use optional instead of | * feat: remove automatic build time __version__ * feat: remove version checking altogether * feat: drop Py3.6 & 3.7 support * ci: remove Py 3.6 & 3.7 * feat: remove multiple import since we're dropping Py3.7 support --------- Co-authored-by: Sofie Van Landeghem --- .github/workflows/tests.yml | 7 +---- .gitignore | 1 - bin/get-package.sh | 12 +++++++++ bin/get-version.sh | 12 +++++++++ bin/push-tag.sh | 19 ++++++++++++++ setup.cfg | 5 ++-- setup.py | 3 ++- weasel/about.py | 1 - weasel/cli/remote_storage.py | 11 +++----- weasel/cli/run.py | 30 ++++----------------- weasel/git_info.py | 3 --- weasel/schemas.py | 20 +++++++++++++- weasel/tests/test_cli_app.py | 51 ++++++++++++++++++++++++++++++++++++ weasel/util.py | 8 ++++-- 14 files changed, 133 insertions(+), 50 deletions(-) create mode 100755 bin/get-package.sh create mode 100755 bin/get-version.sh create mode 100755 bin/push-tag.sh delete mode 100644 weasel/git_info.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 82e3def..c38a470 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -52,12 +52,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python_version: ["3.7", "3.8", "3.9", "3.10", "3.11"] - include: - - os: windows-2019 - python_version: "3.6" - - os: ubuntu-20.04 - python_version: "3.6" + python_version: ["3.8", "3.9", "3.10", "3.11"] runs-on: ${{ matrix.os }} steps: diff --git a/.gitignore b/.gitignore index 4a7952e..0969c96 100644 --- a/.gitignore +++ b/.gitignore @@ -44,7 +44,6 @@ Pipfile.lock *.egg .eggs MANIFEST -spacy/git_info.py # Temporary files *.~* diff --git a/bin/get-package.sh b/bin/get-package.sh new file mode 100755 index 0000000..e638278 --- /dev/null +++ b/bin/get-package.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +version=$(grep "name = " setup.cfg) +version=${version/__title__ = } +version=${version/\'/} +version=${version/\'/} +version=${version/\"/} +version=${version/\"/} + +echo $version diff --git a/bin/get-version.sh b/bin/get-version.sh new file mode 100755 index 0000000..7b263cc --- /dev/null +++ b/bin/get-version.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +version=$(grep "version = " setup.cfg) +version=${version/version = } +version=${version/\'/} +version=${version/\'/} +version=${version/\"/} +version=${version/\"/} + +echo $version diff --git a/bin/push-tag.sh b/bin/push-tag.sh new file mode 100755 index 0000000..4d73207 --- /dev/null +++ b/bin/push-tag.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -e + +# Insist repository is clean +git diff-index --quiet HEAD + +git checkout $1 +git pull origin $1 +git push origin $1 + +version=$(grep "version = " setup.cfg) +version=${version/version = } +version=${version/\'/} +version=${version/\'/} +version=${version/\"/} +version=${version/\"/} +git tag "v$version" +git push origin "v$version" diff --git a/setup.cfg b/setup.cfg index 93a3011..e0dd724 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,6 @@ [metadata] +name = weasel +version = 0.0.0 description = Weasel: A small and easy workflow system url = https://github.com/explosion/weasel/ author = Explosion @@ -15,11 +17,10 @@ classifiers = Operating System :: MacOS :: MacOS X Operating System :: Microsoft :: Windows Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Topic :: Scientific/Engineering project_urls = Release notes = https://github.com/explosion/weasel/releases diff --git a/setup.py b/setup.py index 80081a7..8d88757 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python + if __name__ == "__main__": from setuptools import find_packages, setup - setup(name="weasel", packages=find_packages()) + setup(packages=find_packages()) diff --git a/weasel/about.py b/weasel/about.py index 2a79e2d..14a1760 100644 --- a/weasel/about.py +++ b/weasel/about.py @@ -1,3 +1,2 @@ -__version__ = "3.5.0" __projects__ = "https://github.com/explosion/projects" __projects_branch__ = "v3" diff --git a/weasel/cli/remote_storage.py b/weasel/cli/remote_storage.py index 0d35f85..6cf5baf 100644 --- a/weasel/cli/remote_storage.py +++ b/weasel/cli/remote_storage.py @@ -8,12 +8,10 @@ from wasabi import msg -from .. import about from .._util import download_file, ensure_pathy, get_checksum, get_hash, make_tempdir from .._util import upload_file from ..errors import Errors -from ..git_info import GIT_VERSION -from ..util import ENV_VARS, check_bool_env_var, get_minor_version +from ..util import check_spacy_env_vars if TYPE_CHECKING: from pathy import FluidPath @@ -161,12 +159,9 @@ def get_command_hash( currently installed packages, whatever environment variables have been marked as relevant, and the command. """ - if check_bool_env_var(ENV_VARS.PROJECT_USE_GIT_VERSION): - spacy_v = GIT_VERSION - else: - spacy_v = str(get_minor_version(about.__version__) or "") + check_spacy_env_vars() dep_checksums = [get_checksum(dep) for dep in sorted(deps)] - hashes = [spacy_v, site_hash, env_hash] + dep_checksums + hashes = [site_hash, env_hash] + dep_checksums hashes.extend(cmd) creation_bytes = "".join(hashes).encode("utf8") return hashlib.md5(creation_bytes).hexdigest() diff --git a/weasel/cli/run.py b/weasel/cli/run.py index 1fa5d0c..5be0aa7 100644 --- a/weasel/cli/run.py +++ b/weasel/cli/run.py @@ -8,13 +8,10 @@ from wasabi import msg from wasabi.util import locale_escape -from .. import about from .._util import COMMAND, PROJECT_FILE, PROJECT_LOCK, Arg, Opt, app, get_checksum from .._util import get_hash, load_project_config, parse_config_overrides -from ..git_info import GIT_VERSION -from ..util import ENV_VARS, SimpleFrozenDict, SimpleFrozenList, check_bool_env_var -from ..util import is_cwd, is_minor_version_match, join_command, run_command -from ..util import split_command, working_dir +from ..util import SimpleFrozenDict, SimpleFrozenList, check_spacy_env_vars, is_cwd +from ..util import join_command, run_command, split_command, working_dir @app.command( @@ -102,10 +99,10 @@ def project_run( err_help = "Maybe you forgot to run the 'project assets' command or a previous step?" err_exits = 1 if not dry else None msg.fail(err, err_help, exits=err_exits) - check_spacy_commit = check_bool_env_var(ENV_VARS.PROJECT_USE_GIT_VERSION) + check_spacy_env_vars() with working_dir(project_dir) as current_dir: msg.divider(subcommand) - rerun = check_rerun(current_dir, cmd, check_spacy_commit=check_spacy_commit) + rerun = check_rerun(current_dir, cmd) if not rerun and not force: msg.info(f"Skipping '{cmd['name']}': nothing changed") else: @@ -223,9 +220,6 @@ def validate_subcommand( def check_rerun( project_dir: Path, command: Dict[str, Any], - *, - check_spacy_version: bool = True, - check_spacy_commit: bool = False, ) -> bool: """Check if a command should be rerun because its settings or inputs/outputs changed. @@ -248,23 +242,11 @@ def check_rerun( # Always run commands with no outputs (otherwise they'd always be skipped) if not entry.get("outs", []): return True - # Always rerun if spaCy version or commit hash changed - spacy_v = entry.get("spacy_version") - commit = entry.get("spacy_git_version") - if check_spacy_version and not is_minor_version_match(spacy_v, about.__version__): - info = f"({spacy_v} in {PROJECT_LOCK}, {about.__version__} current)" - msg.info(f"Re-running '{command['name']}': spaCy minor version changed {info}") - return True - if check_spacy_commit and commit != GIT_VERSION: - info = f"({commit} in {PROJECT_LOCK}, {GIT_VERSION} current)" - msg.info(f"Re-running '{command['name']}': spaCy commit changed {info}") - return True # If the entry in the lockfile matches the lockfile entry that would be # generated from the current command, we don't rerun because it means that # all inputs/outputs, hashes and scripts are the same and nothing changed lock_entry = get_lock_entry(project_dir, command) - exclude = ["spacy_version", "spacy_git_version"] - return get_hash(lock_entry, exclude=exclude) != get_hash(entry, exclude=exclude) + return get_hash(lock_entry) != get_hash(entry) def update_lockfile(project_dir: Path, command: Dict[str, Any]) -> None: @@ -303,8 +285,6 @@ def get_lock_entry(project_dir: Path, command: Dict[str, Any]) -> Dict[str, Any] "script": command["script"], "deps": deps, "outs": [*outs, *outs_nc], - "spacy_version": about.__version__, - "spacy_git_version": GIT_VERSION, } diff --git a/weasel/git_info.py b/weasel/git_info.py deleted file mode 100644 index 6cf2d30..0000000 --- a/weasel/git_info.py +++ /dev/null @@ -1,3 +0,0 @@ -# THIS FILE IS COPIED FROM SPACY, WHERE IT WAS GENERATED AUTOMATICALLY -# -GIT_VERSION = "6aa6b86d4" diff --git a/weasel/schemas.py b/weasel/schemas.py index 34a7c79..25c925e 100644 --- a/weasel/schemas.py +++ b/weasel/schemas.py @@ -3,7 +3,8 @@ from collections import defaultdict from typing import Any, Dict, List, Optional, Type, Union -from pydantic import BaseModel, Field, StrictStr, ValidationError, root_validator +from pydantic import BaseModel, BaseSettings, Field, StrictStr, ValidationError +from pydantic import root_validator from wasabi import msg @@ -91,3 +92,20 @@ def check_legacy_keys(cls, obj: Dict[str, Any]) -> Dict[str, Any]: "which is now deprecated. Weasel will not validate your version of spaCy.", ) return obj + + +class SpacyEnvVars(BaseSettings): + SPACY_CONFIG_OVERRIDES: Optional[str] = None + SPACY_PROJECT_USE_GIT_VERSION: Optional[bool] = None + + def check(self): + if self.SPACY_CONFIG_OVERRIDES is not None: + msg.warn( + "You've set a `SPACY_CONFIG_OVERRIDES` environment variable, " + "which is now deprecated. Weasel will not use it." + ) + if self.SPACY_PROJECT_USE_GIT_VERSION is not None: + msg.warn( + "You've set a `SPACY_PROJECT_USE_GIT_VERSION` environment variable, " + "which is now deprecated. Weasel will not use it." + ) diff --git a/weasel/tests/test_cli_app.py b/weasel/tests/test_cli_app.py index 9279679..bbc4c97 100644 --- a/weasel/tests/test_cli_app.py +++ b/weasel/tests/test_cli_app.py @@ -1,3 +1,4 @@ +import os from pathlib import Path from typing import Any, Dict @@ -96,10 +97,60 @@ def test_project_run(project_dir: Path): result = CliRunner().invoke(app, ["run", "create", str(project_dir)]) assert result.exit_code == 0 assert test_file.is_file() + + assert ( + "You've set a `SPACY_CONFIG_OVERRIDES` environment variable" + not in result.output + ) + assert ( + "You've set a `SPACY_PROJECT_USE_GIT_VERSION` environment variable" + not in result.output + ) + + os.environ["SPACY_CONFIG_OVERRIDES"] = "test" + os.environ["SPACY_PROJECT_USE_GIT_VERSION"] = "false" + result = CliRunner().invoke(app, ["run", "ok", str(project_dir)]) assert result.exit_code == 0 assert "okokok" in result.stdout + assert "You've set a `SPACY_CONFIG_OVERRIDES` environment variable" in result.output + assert ( + "You've set a `SPACY_PROJECT_USE_GIT_VERSION` environment variable" + in result.output + ) + + +def test_check_spacy_env_vars(project_dir: Path, monkeypatch: pytest.MonkeyPatch): + # make sure dry run works + project_dir / "abc.txt" + + monkeypatch.delenv("SPACY_CONFIG_OVERRIDES") + monkeypatch.delenv("SPACY_PROJECT_USE_GIT_VERSION") + + result = CliRunner().invoke(app, ["run", "--dry", "create", str(project_dir)]) + assert result.exit_code == 0 + assert ( + "You've set a `SPACY_CONFIG_OVERRIDES` environment variable" + not in result.output + ) + assert ( + "You've set a `SPACY_PROJECT_USE_GIT_VERSION` environment variable" + not in result.output + ) + + monkeypatch.setenv("SPACY_CONFIG_OVERRIDES", "test") + monkeypatch.setenv("SPACY_PROJECT_USE_GIT_VERSION", "false") + + result = CliRunner().invoke(app, ["run", "--dry", "create", str(project_dir)]) + assert result.exit_code == 0 + + assert "You've set a `SPACY_CONFIG_OVERRIDES` environment variable" in result.output + assert ( + "You've set a `SPACY_PROJECT_USE_GIT_VERSION` environment variable" + in result.output + ) + @pytest.mark.skipif(not has_git(), reason="git not installed") @pytest.mark.parametrize( diff --git a/weasel/util.py b/weasel/util.py index f3689ff..c6d904c 100644 --- a/weasel/util.py +++ b/weasel/util.py @@ -20,6 +20,7 @@ from .compat import is_windows from .errors import Errors, Warnings +from .schemas import SpacyEnvVars logger = logging.getLogger("spacy") logger_stream_handler = logging.StreamHandler() @@ -30,8 +31,11 @@ class ENV_VARS: - CONFIG_OVERRIDES = "SPACY_CONFIG_OVERRIDES" - PROJECT_USE_GIT_VERSION = "SPACY_PROJECT_USE_GIT_VERSION" + CONFIG_OVERRIDES = "WEASEL_CONFIG_OVERRIDES" + + +def check_spacy_env_vars(): + SpacyEnvVars().check() class SimpleFrozenDict(dict):