feat: define weasel app/entrypoints (#10)

* feat: add weasel and spacy entrypoints

* feat: enable python -m weasel

* feat: change command to weasel

TODO: project help

* revert: remove spacy entrypoints

* feat: import command functions

* test: add and adapt test_cli_app from spaCy

* feat: remove intermediate "project" command

* Revert "feat: remove intermediate "project" command"

This reverts commit 8c42b66.

Limits the number of changes.

* build: add requirements

* feat: remove intermediate project command

* fix: command name with python -m weasel

* test: remove spaCy-fic assets

setup.cfg

@@ -23,4 +23,20 @@ classifiers =
     Topic :: Scientific/Engineering
 project_urls =
     Release notes = https://github.com/explosion/weasel/releases
-    Source = https://github.com/explosion/weasel/
+    Source = https://github.com/explosion/weasel/
+
+[options]
+install_requires =
+    thinc>=8.1.0,<8.2.0
+    wasabi>=0.9.1,<1.2.0
+    srsly>=2.4.3,<3.0.0
+    catalogue>=2.0.6,<2.1.0
+    typer>=0.3.0,<0.8.0
+    pathy>=0.10.0
+    requests>=2.13.0,<3.0.0
+    pydantic>=1.7.4,!=1.8,!=1.8.1,<1.11.0
+
+
+[options.entry_points]
+console_scripts =
+    weasel = weasel._util:app

weasel/__init__.py

@@ -0,0 +1,11 @@
+from ._util import app  # noqa: F401
+
+# These are the actual functions, NOT the wrapped CLI commands. The CLI commands
+# are registered automatically and won't have to be imported here.
+from .cli.assets import project_assets  # noqa: F401
+from .cli.clone import project_clone  # noqa: F401
+from .cli.document import project_document  # noqa: F401
+from .cli.dvc import project_update_dvc  # noqa: F401
+from .cli.pull import project_pull  # noqa: F401
+from .cli.push import project_push  # noqa: F401
+from .cli.run import project_run  # noqa: F401

weasel/__main__.py

@@ -0,0 +1,3 @@
+from ._util import COMMAND, app
+
+app(prog_name=COMMAND)

weasel/_util.py

@@ -27,31 +27,23 @@
     from pathy import FluidPath  # noqa: F401
 
 
-COMMAND = "python -m spacy"
-NAME = "spacy"
-HELP = """spaCy Command-line Interface
+COMMAND = "python -m weasel"
+NAME = "weasel"
+HELP = """weasel Command-line Interface
 
 DOCS: https://spacy.io/api/cli
 """
 
 PROJECT_FILE = "project.yml"
 PROJECT_LOCK = "project.lock"
 
-PROJECT_HELP = f"""Command-line interface for spaCy projects and templates.
-You'd typically start by cloning a project template to a local directory and
-fetching its assets like datasets etc. See the project's {PROJECT_FILE} for the
-available commands.
-"""
-
 
 # Wrappers for Typer's annotations. Initially created to set defaults and to
 # keep the names short, but not needed at the moment.
 Arg = typer.Argument
 Opt = typer.Option
 
 app = typer.Typer(name=NAME, help=HELP)
-project_cli = typer.Typer(name="project", help=PROJECT_HELP, no_args_is_help=True)
-app.add_typer(project_cli)
 
 
 def parse_config_overrides(

weasel/cli/assets.py

@@ -8,15 +8,15 @@
 import typer
 
 from ..util import ensure_path, working_dir
-from .._util import project_cli, Arg, Opt, PROJECT_FILE, load_project_config
+from .._util import app, Arg, Opt, PROJECT_FILE, load_project_config
 from .._util import get_checksum, download_file, git_checkout, get_git_version
 from .._util import SimpleFrozenDict, parse_config_overrides
 
 # Whether assets are extra if `extra` is not set.
 EXTRA_DEFAULT = False
 
 
-@project_cli.command(
+@app.command(
     "assets",
     context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
 )

weasel/cli/clone.py

@@ -6,15 +6,15 @@
 
 from .. import about
 from ..util import ensure_path
-from .._util import project_cli, Arg, Opt, COMMAND, PROJECT_FILE
+from .._util import app, Arg, Opt, COMMAND, PROJECT_FILE
 from .._util import git_checkout, get_git_version, git_repo_branch_exists
 
 DEFAULT_REPO = about.__projects__
 DEFAULT_PROJECTS_BRANCH = about.__projects_branch__
 DEFAULT_BRANCHES = ["main", "master"]
 
 
-@project_cli.command("clone")
+@app.command("clone")
 def project_clone_cli(
     # fmt: off
     name: str = Arg(..., help="The name of the template to clone"),

weasel/cli/document.py

@@ -2,7 +2,7 @@
 from wasabi import msg, MarkdownRenderer
 
 from ..util import working_dir
-from .._util import project_cli, Arg, Opt, PROJECT_FILE, load_project_config
+from .._util import app, Arg, Opt, PROJECT_FILE, load_project_config
 
 
 DOCS_URL = "https://spacy.io"
@@ -27,7 +27,7 @@
 MARKER_IGNORE = "<!-- SPACY PROJECT: IGNORE -->"
 
 
-@project_cli.command("document")
+@app.command("document")
 def project_document_cli(
     # fmt: off
     project_dir: Path = Arg(Path.cwd(), help="Path to cloned project. Defaults to current working directory.", exists=True, file_okay=False),

weasel/cli/dvc.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from wasabi import msg
 
-from .._util import PROJECT_FILE, load_project_config, get_hash, project_cli
+from .._util import PROJECT_FILE, load_project_config, get_hash, app
 from .._util import Arg, Opt, NAME, COMMAND
 from ..util import working_dir, split_command, join_command, run_command
 from ..util import SimpleFrozenList
@@ -19,7 +19,7 @@
 # {COMMAND} project {UPDATE_COMMAND}"""
 
 
-@project_cli.command(UPDATE_COMMAND)
+@app.command(UPDATE_COMMAND)
 def project_update_dvc_cli(
     # fmt: off
     project_dir: Path = Arg(Path.cwd(), help="Location of project directory. Defaults to current working directory.", exists=True, file_okay=False),

weasel/cli/pull.py

@@ -2,12 +2,12 @@
 from wasabi import msg
 from .remote_storage import RemoteStorage
 from .remote_storage import get_command_hash
-from .._util import project_cli, Arg, logger
+from .._util import app, Arg, logger
 from .._util import load_project_config
 from .run import update_lockfile
 
 
-@project_cli.command("pull")
+@app.command("pull")
 def project_pull_cli(
     # fmt: off
     remote: str = Arg("default", help="Name or path of remote storage"),

weasel/cli/push.py

@@ -3,10 +3,10 @@
 from .remote_storage import RemoteStorage
 from .remote_storage import get_content_hash, get_command_hash
 from .._util import load_project_config
-from .._util import project_cli, Arg, logger
+from .._util import app, Arg, logger
 
 
-@project_cli.command("push")
+@app.command("push")
 def project_push_cli(
     # fmt: off
     remote: str = Arg("default", help="Name or path of remote storage"),

weasel/cli/run.py

@@ -14,10 +14,10 @@
 from ..util import SimpleFrozenList, is_minor_version_match, ENV_VARS
 from ..util import check_bool_env_var, SimpleFrozenDict
 from .._util import PROJECT_FILE, PROJECT_LOCK, load_project_config, get_hash
-from .._util import get_checksum, project_cli, Arg, Opt, COMMAND, parse_config_overrides
+from .._util import get_checksum, app, Arg, Opt, COMMAND, parse_config_overrides
 
 
-@project_cli.command(
+@app.command(
     "run", context_settings={"allow_extra_args": True, "ignore_unknown_options": True}
 )
 def project_run_cli(

weasel/tests/test_cli_app.py

@@ -0,0 +1,148 @@
+from pathlib import Path
+
+import pytest
+import srsly
+from typer.testing import CliRunner
+
+from weasel._util import app, get_git_version
+
+
+def has_git():
+    try:
+        get_git_version()
+        return True
+    except RuntimeError:
+        return False
+
+
+SAMPLE_PROJECT = {
+    "title": "Sample project",
+    "description": "This is a project for testing",
+    "assets": [
+        {
+            "dest": "assets/weasel-readme.md",
+            "url": "https://github.com/explosion/weasel/raw/9a3632862b47069d2f9033b773e814d4c4e09c83/README.md",
+            "checksum": "65f4c426a9b153b7683738c92d0d20f9",
+        },
+        {
+            "dest": "assets/pyproject.toml",
+            "url": "https://github.com/explosion/weasel/raw/9a3632862b47069d2f9033b773e814d4c4e09c83/pyproject.toml",
+            "checksum": "1e2da3a3030d6611520952d5322cd94e",
+            "extra": True,
+        },
+    ],
+    "commands": [
+        {
+            "name": "ok",
+            "help": "print ok",
+            "script": ["python -c \"print('okokok')\""],
+        },
+        {
+            "name": "create",
+            "help": "make a file",
+            "script": ["touch abc.txt"],
+            "outputs": ["abc.txt"],
+        },
+        {
+            "name": "clean",
+            "help": "remove test file",
+            "script": ["rm abc.txt"],
+        },
+    ],
+}
+
+SAMPLE_PROJECT_TEXT = srsly.yaml_dumps(SAMPLE_PROJECT)
+
+
+@pytest.fixture
+def project_dir(tmp_path: Path):
+    path = tmp_path / "project"
+    path.mkdir()
+    (path / "project.yml").write_text(SAMPLE_PROJECT_TEXT)
+    yield path
+
+
+def test_project_document(project_dir: Path):
+    readme_path = project_dir / "README.md"
+    assert not readme_path.exists(), "README already exists"
+    result = CliRunner().invoke(
+        app, ["document", str(project_dir), "-o", str(readme_path)]
+    )
+    assert result.exit_code == 0
+    assert readme_path.is_file()
+    text = readme_path.read_text("utf-8")
+    assert SAMPLE_PROJECT["description"] in text
+
+
+def test_project_assets(project_dir: Path):
+    asset_dir = project_dir / "assets"
+    assert not asset_dir.exists(), "Assets dir is already present"
+    result = CliRunner().invoke(app, ["assets", str(project_dir)])
+    assert result.exit_code == 0
+    assert (asset_dir / "weasel-readme.md").is_file(), "Assets not downloaded"
+    # check that extras work
+    result = CliRunner().invoke(app, ["assets", "--extra", str(project_dir)])
+    assert result.exit_code == 0
+    assert (asset_dir / "pyproject.toml").is_file(), "Extras not downloaded"
+
+
+def test_project_run(project_dir: Path):
+    # make sure dry run works
+    test_file = project_dir / "abc.txt"
+    result = CliRunner().invoke(app, ["run", "--dry", "create", str(project_dir)])
+    assert result.exit_code == 0
+    assert not test_file.is_file()
+    result = CliRunner().invoke(app, ["run", "create", str(project_dir)])
+    assert result.exit_code == 0
+    assert test_file.is_file()
+    result = CliRunner().invoke(app, ["run", "ok", str(project_dir)])
+    assert result.exit_code == 0
+    assert "okokok" in result.stdout
+
+
+@pytest.mark.skipif(not has_git(), reason="git not installed")
+@pytest.mark.parametrize(
+    "options",
+    [
+        "",
+        # "--sparse",
+        "--branch v3",
+        "--repo https://github.com/explosion/projects --branch v3",
+    ],
+)
+def test_project_clone(tmp_path: Path, options: str):
+
+    out = tmp_path / "project_clone"
+    target = "benchmarks/ner_conll03"
+    if not options:
+        options = []
+    else:
+        options = options.split()
+    result = CliRunner().invoke(app, ["clone", target, *options, str(out)])
+    assert result.exit_code == 0
+    assert (out / "README.md").is_file()
+
+
+def test_project_push_pull(tmp_path: Path, project_dir: Path):
+    proj = dict(SAMPLE_PROJECT)
+    remote = "xyz"
+
+    remote_dir = tmp_path / "remote"
+    remote_dir.mkdir()
+
+    proj["remotes"] = {remote: str(remote_dir)}
+    proj_text = srsly.yaml_dumps(proj)
+    (project_dir / "project.yml").write_text(proj_text)
+
+    test_file = project_dir / "abc.txt"
+    result = CliRunner().invoke(app, ["run", "create", str(project_dir)])
+    assert result.exit_code == 0
+    assert test_file.is_file()
+    result = CliRunner().invoke(app, ["push", remote, str(project_dir)])
+    assert result.exit_code == 0
+    result = CliRunner().invoke(app, ["run", "clean", str(project_dir)])
+    assert result.exit_code == 0
+    assert not test_file.exists()
+    result = CliRunner().invoke(app, ["pull", remote, str(project_dir)])
+    assert result.exit_code == 0
+    assert test_file.is_file()

weasel/tests/util.py

@@ -0,0 +1,26 @@
+import contextlib
+import re
+import tempfile
+
+import srsly
+
+
+@contextlib.contextmanager
+def make_tempfile(mode="r"):
+    f = tempfile.TemporaryFile(mode=mode)
+    yield f
+    f.close()
+
+
+def assert_packed_msg_equal(b1, b2):
+    """Assert that two packed msgpack messages are equal."""
+    msg1 = srsly.msgpack_loads(b1)
+    msg2 = srsly.msgpack_loads(b2)
+    assert sorted(msg1.keys()) == sorted(msg2.keys())
+    for (k1, v1), (k2, v2) in zip(sorted(msg1.items()), sorted(msg2.items())):
+        assert k1 == k2
+        assert v1 == v2
+
+
+def normalize_whitespace(s):
+    return re.sub(r"\s+", " ", s)