diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..fce789feb --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,101 @@ +version: 2.1 + +parameters: + REF: + type: string + default: '' + description: Optional tag to build + +jobs: + arm-wheels: + parameters: + build: + type: string + image: + type: string + + machine: + image: ubuntu-2404:current + resource_class: arm.medium # 2 vCPUs + + environment: + CIBW_ARCHS: "aarch64" + CIBW_MANYLINUX_AARCH64_IMAGE: "<< parameters.image >>" + CIBW_MUSLLINUX_AARCH64_IMAGE: "<< parameters.image >>" + CIBW_BUILD: "<< parameters.build >>" + + steps: + - checkout + - when: + condition: << pipeline.parameters.REF >> + steps: + - run: + name: Checkout branch/tag << pipeline.parameters.REF >> + command: | + echo "Switching to branch/tag << pipeline.parameters.REF >> if it exists" + git checkout << pipeline.parameters.REF >> || true + git pull origin << pipeline.parameters.REF >> || true + - run: + name: install cibuildwheel and other build reqs + command: | + python3 -m pip install --upgrade pip setuptools setuptools_scm[toml] + python3 -m pip install -rcibw-requirements.txt + + - run: + name: pip freeze + command: | + python3 -m pip freeze + + - run: + name: list wheels + command: | + python3 -m cibuildwheel . --print-build-identifiers + + - run: + name: cibuildwheel + command: | + python3 -m cibuildwheel . + + - store_test_results: + path: test-results/ + + - store_artifacts: + path: wheelhouse/ + + # - when: + # condition: + # or: + # - matches: + # pattern: ".+" + # value: "<< pipeline.git.tag >>" + # - << pipeline.parameters.REF >> + # steps: + # - run: + # environment: + # TWINE_NONINTERACTIVE: "1" + # command: | + # python3 -m pip install twine + # python3 -m twine upload --verbose --skip-existing wheelhouse/* + +workflows: + wheels: # This is the name of the workflow, feel free to change it to better match your workflow. + # Inside the workflow, you define the jobs you want to run. + jobs: + - arm-wheels: + name: arm-wheels-manylinux_2_28 + filters: + branches: + only: main + tags: + only: /.*/ + build: "*manylinux*" + image: quay.io/pypa/manylinux_2_28_aarch64 + - arm-wheels: + name: arm-wheels-musllinux_1_2 + filters: + branches: + only: main + tags: + only: /.*/ + build: "*musllinux*" + image: quay.io/pypa/musllinux_1_2_aarch64 diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 1530ab92b..1295eebf1 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -32,7 +32,7 @@ jobs: strategy: matrix: py-ver-major: [3] - py-ver-minor: [8, 9, 10, 11, 12, 13] + py-ver-minor: [9, 10, 11, 12, 13] step: [lint, unit, bandit, mypy] env: @@ -44,11 +44,11 @@ jobs: with: fetch-depth: 0 - - name: Set up Singularity + - name: Set up Singularity and environment-modules if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} @@ -81,7 +81,7 @@ jobs: - name: Upload coverage to Codecov if: ${{ matrix.step == 'unit' }} - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: fail_ci_if_error: true env: @@ -89,7 +89,7 @@ jobs: tox-style: name: Linters - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 strategy: matrix: @@ -132,10 +132,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Singularity + - name: Set up Singularity and environment-modules run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. run: sudo usermod -c 'CI Runner' "$(whoami)" @@ -157,7 +157,7 @@ jobs: chmod a-w . - name: run tests - run: APPTAINER_TMPDIR=${RUNNER_TEMP} make test + run: APPTAINER_TMPDIR=${RUNNER_TEMP} make test PYTEST_EXTRA=-vvv conformance_tests: @@ -165,6 +165,7 @@ jobs: runs-on: ubuntu-22.04 strategy: + fail-fast: false matrix: cwl-version: [v1.0, v1.1, v1.2] container: [docker, singularity, podman] @@ -179,11 +180,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Singularity + - name: Set up Singularity and environment-modules if: ${{ matrix.container == 'singularity' }} run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Singularity cache if: ${{ matrix.container == 'singularity' }} @@ -216,7 +217,7 @@ jobs: path: | **/cwltool_conf*.xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: fail_ci_if_error: true env: @@ -228,10 +229,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Singularity + - name: Set up Singularity and environment-modules run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules - name: Set up Python uses: actions/setup-python@v5 @@ -301,7 +302,7 @@ jobs: - name: Test with tox run: tox - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: fail_ci_if_error: true env: diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 000000000..ed10cd17d --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,127 @@ +name: Python package build and publish + +on: + release: + types: [published] + workflow_dispatch: {} + repository_dispatch: {} + push: + branches: + - main + +concurrency: + group: wheels-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build_wheels: + name: ${{ matrix.image }} wheels + runs-on: ubuntu-24.04 + strategy: + matrix: + include: + - image: manylinux_2_28_x86_64 + build: "*manylinux*" + - image: musllinux_1_2_x86_64 + build: "*musllinux*" + + steps: + - uses: actions/checkout@v4 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v4 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + # - name: Set up QEMU + # if: runner.os == 'Linux' + # uses: docker/setup-qemu-action@v2 + # with: + # platforms: all + + - name: Build wheels + uses: pypa/cibuildwheel@v2.22.0 + env: + CIBW_BUILD: ${{ matrix.build }} + CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} + CIBW_MUSLLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} + # configure cibuildwheel to build native 64-bit archs ('auto64'), and some + # emulated ones + # Linux arm64 wheels are built on circleci + CIBW_ARCHS_LINUX: auto64 # ppc64le s390x + + - uses: actions/upload-artifact@v4 + with: + name: artifact-${{ matrix.image }} + path: ./wheelhouse/*.whl + + build_sdist: + name: Build source distribution + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v4 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + - name: Build sdist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v4 + with: + name: artifact-source + path: dist/*.tar.gz + + build_wheels_macos: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + # macos-13 is an intel runner, macos-14 is apple silicon + os: [macos-13, macos-14] + steps: + - uses: actions/checkout@v4 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v4 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + - name: Build wheels + uses: pypa/cibuildwheel@v2.22.0 + + - uses: actions/upload-artifact@v4 + with: + name: artifact-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl + + # upload_pypi: + # needs: [build_wheels, build_sdist] + # runs-on: ubuntu-24.04 + # environment: deploy + # permissions: + # id-token: write + # if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'repository_dispatch' && github.event.client_payload.publish_wheel == true) + # steps: + # - uses: actions/download-artifact@v4 + # with: + # # unpacks default artifact into dist/ + # pattern: artifact-* + # merge-multiple: true + # path: dist + + # - name: Publish package distributions to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # skip-existing: true diff --git a/.gitignore b/.gitignore index fbe4b24fc..b4cab0e66 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ eggs/ *.egg .tox/ .pytest_cache +*.so # Editor Temps .*.sw? diff --git a/MANIFEST.in b/MANIFEST.in index 187d19bea..83b8c3fa5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -11,6 +11,7 @@ include tests/loop-ext/* include tests/tmp1/tmp2/tmp3/.gitkeep include tests/tmp4/alpha/* include tests/wf/* +include tests/wf/inp-filelist.txt include tests/wf/operation/* include tests/override/* include tests/reloc/*.cwl @@ -19,6 +20,7 @@ include tests/reloc/dir2/* include tests/checker_wf/* include tests/subgraph/* include tests/input_deps/* +recursive-include tests/test_deps_env include tests/trs/* include tests/wf/generator/* include cwltool/py.typed diff --git a/Makefile b/Makefile index 1b08f4290..5b9dd214e 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ MODULE=cwltool # `SHELL=bash` doesn't work for some, so don't use BASH-isms like # `[[` conditional expressions. -PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py) setup.py +PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py tests/cwl-conformance/*.py) setup.py DEVPKGS=diff_cover pylint pep257 pydocstyle 'tox<4' tox-pyenv auto-walrus \ isort wheel autoflake pyupgrade bandit -rlint-requirements.txt\ -rtest-requirements.txt -rmypy-requirements.txt -rdocs/requirements.txt @@ -190,7 +190,7 @@ shellcheck: FORCE cwltool-in-docker.sh pyupgrade: $(PYSOURCES) - pyupgrade --exit-zero-even-if-changed --py38-plus $^ + pyupgrade --exit-zero-even-if-changed --py39-plus $^ auto-walrus $^ release-test: FORCE diff --git a/README.rst b/README.rst index db40d0420..7c6ffe22d 100644 --- a/README.rst +++ b/README.rst @@ -52,7 +52,7 @@ and provide comprehensive validation of CWL files as well as provide other tools related to working with CWL. ``cwltool`` is written and tested for -`Python `_ ``3.x {x = 6, 8, 9, 10, 11}`` +`Python `_ ``3.x {x = 9, 10, 11, 12, 13}`` The reference implementation consists of two packages. The ``cwltool`` package is the primary Python module containing the reference implementation in the @@ -189,6 +189,22 @@ and ``--tmp-outdir-prefix`` to somewhere under ``/Users``:: $ cwl-runner --tmp-outdir-prefix=/Users/username/project --tmpdir-prefix=/Users/username/project wc-tool.cwl wc-job.json + +Docker default platform on macOS with Apple Silicon +=================================================== + +If macOS users want to run CWL tools/workflows using ``cwltool`` with Docker and their software containers only have support for amd64 (64-bit x86) CPUs, but they have an Apple Silicon (aarch64/arm64) CPU, +they run into the error: + + WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested. + +To fix this, export the following environment variable before executing `cwltool`: + +``export DOCKER_DEFAULT_PLATFORM=linux/amd64`` + +To automatically have this variable set in the future, add the same command to ones respective shell profile (e.g. ``~/.zshrc``, ``~/.bash_profile``). + + Using uDocker ============= diff --git a/build-cwltool-docker.sh b/build-cwltool-docker.sh index a70fdf4df..9f7163afb 100755 --- a/build-cwltool-docker.sh +++ b/build-cwltool-docker.sh @@ -8,4 +8,4 @@ ${engine} run -t -v /var/run/docker.sock:/var/run/docker.sock \ -v /tmp:/tmp \ -v "$PWD":/tmp/cwltool \ quay.io/commonwl/cwltool_module /bin/sh -c \ - "apk add gcc bash git && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not (test_bioconda or test_double_overwrite or test_env_filtering or test_biocontainers or test_disable_file_overwrite_without_ext or test_disable_file_creation_in_outdir_with_ext or test_write_write_conflict or test_directory_literal_with_real_inputs_inside or test_revsort_workflow or test_stdin_with_id_preset or test_no_compute_chcksum or test_packed_workflow_execution[tests/wf/count-lines1-wf.cwl-tests/wf/wc-job.json-False] or test_sequential_workflow or test_single_process_subwf_subwf_inline_step)' --ignore-glob '*test_udocker.py' -n 0 -v -rs --pyargs cwltool" + "apk add gcc bash git && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not (test_bioconda or test_double_overwrite or test_env_filtering or test_biocontainers or test_disable_file_overwrite_without_ext or test_disable_file_creation_in_outdir_with_ext or test_write_write_conflict or test_directory_literal_with_real_inputs_inside or test_revsort_workflow or test_stdin_with_id_preset or test_no_compute_chcksum or test_packed_workflow_execution[tests/wf/count-lines1-wf.cwl-tests/wf/wc-job.json-False] or test_sequential_workflow or test_single_process_subwf_subwf_inline_step or test_cache_dockerreq_hint_instead_of_req)' --ignore-glob '*test_udocker.py' -n 0 -v -rs --pyargs cwltool" diff --git a/cibw-requirements.txt b/cibw-requirements.txt new file mode 100644 index 000000000..833aca23d --- /dev/null +++ b/cibw-requirements.txt @@ -0,0 +1 @@ +cibuildwheel==2.22.0 diff --git a/conformance-test.sh b/conformance-test.sh index 36ea23b17..6df14a63c 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -97,19 +97,7 @@ if [[ "$VERSION" = *dev* ]] then CWLTOOL_OPTIONS+=" --enable-dev" fi -if [[ "$CONTAINER" = "singularity" ]]; then - CWLTOOL_OPTIONS+=" --singularity" - # This test fails because Singularity and Docker have - # different views on how to deal with this. - exclusions+=(docker_entrypoint) - if [[ "${VERSION}" = "v1.1" ]]; then - # This fails because of a difference (in Singularity vs Docker) in - # the way filehandles are passed to processes in the container and - # wc can tell somehow. - # See issue #1440 - exclusions+=(stdin_shorcut) - fi -elif [[ "$CONTAINER" = "podman" ]]; then +if [[ "$CONTAINER" = "podman" ]]; then CWLTOOL_OPTIONS+=" --podman" fi diff --git a/cwltool/argparser.py b/cwltool/argparser.py index efced5386..3f07cea9d 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -3,19 +3,11 @@ import argparse import os import urllib -from typing import ( - Any, - Callable, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Sequence, - Type, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence, Sequence +from typing import Any, Callable, Optional, Union, cast + +import rich.markup +from rich_argparse import HelpPreviewAction, RichHelpFormatter from .loghandler import _logger from .process import Process, shortname @@ -25,9 +17,11 @@ def arg_parser() -> argparse.ArgumentParser: + RichHelpFormatter.group_name_formatter = str parser = argparse.ArgumentParser( + formatter_class=RichHelpFormatter, description="Reference executor for Common Workflow Language standards. " - "Not for production use." + "Not for production use.", ) parser.add_argument("--basedir", type=str) parser.add_argument( @@ -37,23 +31,15 @@ def arg_parser() -> argparse.ArgumentParser: help="Output directory. The default is the current directory.", ) - parser.add_argument( - "--log-dir", - type=str, - default="", - help="Log your tools stdout/stderr to this location outside of container " - "This will only log stdout/stderr if you specify stdout/stderr in their " - "respective fields or capture it as an output", - ) - parser.add_argument( "--parallel", action="store_true", default=False, help="Run jobs in parallel. ", ) - envgroup = parser.add_mutually_exclusive_group() - envgroup.add_argument( + envgroup = parser.add_argument_group(title="Control environment variables") + env_exclusive = envgroup.add_mutually_exclusive_group() + env_exclusive.add_argument( "--preserve-environment", type=str, action="append", @@ -64,7 +50,7 @@ def arg_parser() -> argparse.ArgumentParser: default=[], dest="preserve_environment", ) - envgroup.add_argument( + env_exclusive.add_argument( "--preserve-entire-environment", action="store_true", help="Preserve all environment variables when running CommandLineTools " @@ -73,54 +59,10 @@ def arg_parser() -> argparse.ArgumentParser: dest="preserve_entire_environment", ) - containergroup = parser.add_mutually_exclusive_group() - containergroup.add_argument( - "--rm-container", - action="store_true", - default=True, - help="Delete Docker container used by jobs after they exit (default)", - dest="rm_container", - ) - - containergroup.add_argument( - "--leave-container", - action="store_false", - default=True, - help="Do not delete Docker container used by jobs after they exit", - dest="rm_container", - ) - - cidgroup = parser.add_argument_group( - "Options for recording the Docker container identifier into a file." - ) - cidgroup.add_argument( - # Disabled as containerid is now saved by default - "--record-container-id", - action="store_true", - default=False, - help=argparse.SUPPRESS, - dest="record_container_id", + files_group = parser.add_argument_group( + title="Manage intermediate, temporary, or final output files" ) - - cidgroup.add_argument( - "--cidfile-dir", - type=str, - help="Store the Docker container ID into a file in the specified directory.", - default=None, - dest="cidfile_dir", - ) - - cidgroup.add_argument( - "--cidfile-prefix", - type=str, - help="Specify a prefix to the container ID filename. " - "Final file name will be followed by a timestamp. " - "The default is no prefix.", - default=None, - dest="cidfile_prefix", - ) - - parser.add_argument( + files_group.add_argument( "--tmpdir-prefix", type=str, help="Path prefix for temporary directories. If --tmpdir-prefix is not " @@ -130,7 +72,7 @@ def arg_parser() -> argparse.ArgumentParser: default=DEFAULT_TMP_PREFIX, ) - intgroup = parser.add_mutually_exclusive_group() + intgroup = files_group.add_mutually_exclusive_group() intgroup.add_argument( "--tmp-outdir-prefix", type=str, @@ -148,7 +90,7 @@ def arg_parser() -> argparse.ArgumentParser: "troubleshooting of CWL documents.", ) - tmpgroup = parser.add_mutually_exclusive_group() + tmpgroup = files_group.add_mutually_exclusive_group() tmpgroup.add_argument( "--rm-tmpdir", action="store_true", @@ -165,7 +107,7 @@ def arg_parser() -> argparse.ArgumentParser: dest="rm_tmpdir", ) - outgroup = parser.add_mutually_exclusive_group() + outgroup = files_group.add_mutually_exclusive_group() outgroup.add_argument( "--move-outputs", action="store_const", @@ -195,30 +137,6 @@ def arg_parser() -> argparse.ArgumentParser: dest="move_outputs", ) - pullgroup = parser.add_mutually_exclusive_group() - pullgroup.add_argument( - "--enable-pull", - default=True, - action="store_true", - help="Try to pull Docker images", - dest="pull_image", - ) - - pullgroup.add_argument( - "--disable-pull", - default=True, - action="store_false", - help="Do not try to pull Docker images", - dest="pull_image", - ) - - parser.add_argument( - "--rdf-serializer", - help="Output RDF serialization format used by --print-rdf (one of " - "turtle (default), n3, nt, xml)", - default="turtle", - ) - parser.add_argument( "--eval-timeout", help="Time to wait for a Javascript expression to evaluate before giving " @@ -227,9 +145,7 @@ def arg_parser() -> argparse.ArgumentParser: default=60, ) - provgroup = parser.add_argument_group( - "Options for recording provenance information of the execution" - ) + provgroup = parser.add_argument_group("Recording provenance information of the execution") provgroup.add_argument( "--provenance", help="Save provenance to specified folder as a " @@ -287,7 +203,8 @@ def arg_parser() -> argparse.ArgumentParser: type=str, ) - printgroup = parser.add_mutually_exclusive_group() + non_exec_group = parser.add_argument_group(title="Non-execution options") + printgroup = non_exec_group.add_mutually_exclusive_group() printgroup.add_argument( "--print-rdf", action="store_true", @@ -335,6 +252,15 @@ def arg_parser() -> argparse.ArgumentParser: printgroup.add_argument( "--make-template", action="store_true", help="Generate a template input object" ) + non_exec_group.add_argument( + "--rdf-serializer", + help="Output RDF serialization format used by --print-rdf (one of " + "turtle (default), n3, nt, xml)", + default="turtle", + ) + non_exec_group.add_argument( + "--tool-help", action="store_true", help="Print command line help for tool" + ) strictgroup = parser.add_mutually_exclusive_group() strictgroup.add_argument( @@ -376,11 +302,27 @@ def arg_parser() -> argparse.ArgumentParser: dest="doc_cache", ) - volumegroup = parser.add_mutually_exclusive_group() - volumegroup.add_argument("--verbose", action="store_true", help="Default logging") - volumegroup.add_argument("--no-warnings", action="store_true", help="Only print errors.") - volumegroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.") - volumegroup.add_argument("--debug", action="store_true", help="Print even more logging") + volumegroup = parser.add_argument_group(title="Configure logging") + volume_exclusive = volumegroup.add_mutually_exclusive_group() + volume_exclusive.add_argument("--verbose", action="store_true", help="Default logging") + volume_exclusive.add_argument("--no-warnings", action="store_true", help="Only print errors.") + volume_exclusive.add_argument( + "--quiet", action="store_true", help="Only print warnings and errors." + ) + volume_exclusive.add_argument("--debug", action="store_true", help="Print even more logging") + volumegroup.add_argument( + "--log-dir", + type=str, + default="", + help="Log your tools stdout/stderr to this location outside of container " + "This will only log stdout/stderr if you specify stdout/stderr in their " + "respective fields or capture it as an output", + ) + volumegroup.add_argument( + "--timestamps", + action="store_true", + help="Add timestamps to the errors, warnings, and notifications.", + ) parser.add_argument( "--write-summary", @@ -391,30 +333,6 @@ def arg_parser() -> argparse.ArgumentParser: dest="write_summary", ) - parser.add_argument( - "--strict-memory-limit", - action="store_true", - help="When running with " - "software containers and the Docker engine, pass either the " - "calculated memory allocation from ResourceRequirements or the " - "default of 1 gigabyte to Docker's --memory option.", - ) - - parser.add_argument( - "--strict-cpu-limit", - action="store_true", - help="When running with " - "software containers and the Docker engine, pass either the " - "calculated cpu allocation from ResourceRequirements or the " - "default of 1 core to Docker's --cpu option. " - "Requires docker version >= v1.13.", - ) - - parser.add_argument( - "--timestamps", - action="store_true", - help="Add timestamps to the errors, warnings, and notifications.", - ) parser.add_argument( "--js-console", action="store_true", help="Enable javascript console output" ) @@ -429,7 +347,103 @@ def arg_parser() -> argparse.ArgumentParser: help="File of options to pass to jshint. " 'This includes the added option "includewarnings". ', ) - dockergroup = parser.add_mutually_exclusive_group() + container_group = parser.add_argument_group( + title="Software container engine selection and configuration" + ) + pullgroup = container_group.add_mutually_exclusive_group() + pullgroup.add_argument( + "--enable-pull", + default=True, + action="store_true", + help="Try to pull Docker images", + dest="pull_image", + ) + + pullgroup.add_argument( + "--disable-pull", + default=True, + action="store_false", + help="Do not try to pull Docker images", + dest="pull_image", + ) + container_group.add_argument( + "--force-docker-pull", + action="store_true", + default=False, + help="Pull latest software container image even if it is locally present", + dest="force_docker_pull", + ) + container_group.add_argument( + "--no-read-only", + action="store_true", + default=False, + help="Do not set root directory in the container as read-only", + dest="no_read_only", + ) + + container_group.add_argument( + "--default-container", + help="Specify a default software container to use for any " + "CommandLineTool without a DockerRequirement.", + ) + container_group.add_argument( + "--no-match-user", + action="store_true", + help="Disable passing the current uid to `docker run --user`", + ) + container_group.add_argument( + "--custom-net", + type=str, + help="Passed to `docker run` as the `--net` parameter when " + "NetworkAccess is true, which is its default setting.", + ) + + container_cleanup = container_group.add_mutually_exclusive_group() + container_cleanup.add_argument( + "--rm-container", + action="store_true", + default=True, + help="Delete Docker container used by jobs after they exit (default)", + dest="rm_container", + ) + + container_cleanup.add_argument( + "--leave-container", + action="store_false", + default=True, + help="Do not delete Docker container used by jobs after they exit", + dest="rm_container", + ) + + cidgroup = parser.add_argument_group("Recording the software container identifier into a file") + cidgroup.add_argument( + # Disabled as containerid is now saved by default + "--record-container-id", + action="store_true", + default=False, + help=argparse.SUPPRESS, + dest="record_container_id", + ) + + cidgroup.add_argument( + "--cidfile-dir", + type=str, + help="Store the software container ID into a file in the specified directory.", + default=None, + dest="cidfile_dir", + ) + + cidgroup.add_argument( + "--cidfile-prefix", + type=str, + help="Specify a prefix to the software container ID filename. " + "Final file name will be followed by a timestamp. " + "The default is no prefix.", + default=None, + dest="cidfile_prefix", + ) + + dockergroup = container_group.add_mutually_exclusive_group() dockergroup.add_argument( "--user-space-docker-cmd", metavar="CMD", @@ -469,6 +483,24 @@ def arg_parser() -> argparse.ArgumentParser: "is specified under `hints`.", dest="use_container", ) + container_group.add_argument( + "--strict-memory-limit", + action="store_true", + help="When running with " + "software containers and the Docker engine, pass either the " + "calculated memory allocation from ResourceRequirements or the " + "default of 1 gigabyte to Docker's --memory option.", + ) + + container_group.add_argument( + "--strict-cpu-limit", + action="store_true", + help="When running with " + "software containers and the Docker engine, pass either the " + "calculated cpu allocation from ResourceRequirements or the " + "default of 1 core to Docker's --cpu option. " + "Requires docker version >= v1.13.", + ) dependency_resolvers_configuration_help = argparse.SUPPRESS dependencies_directory_help = argparse.SUPPRESS @@ -478,7 +510,7 @@ def arg_parser() -> argparse.ArgumentParser: if SOFTWARE_REQUIREMENTS_ENABLED: dependency_resolvers_configuration_help = ( "Dependency resolver " - "configuration file describing how to adapt 'SoftwareRequirement' " + "configuration file describing how to adapt `SoftwareRequirement` " "packages to current system." ) dependencies_directory_help = ( @@ -487,7 +519,7 @@ def arg_parser() -> argparse.ArgumentParser: use_biocontainers_help = ( "Use biocontainers for tools without an " "explicitly annotated Docker container." ) - conda_dependencies = "Short cut to use Conda to resolve 'SoftwareRequirement' packages." + conda_dependencies = "Short cut to use Conda to resolve `SoftwareRequirement` packages." parser.add_argument( "--beta-dependency-resolvers-configuration", @@ -510,8 +542,6 @@ def arg_parser() -> argparse.ArgumentParser: action="store_true", ) - parser.add_argument("--tool-help", action="store_true", help="Print command line help for tool") - parser.add_argument( "--relative-deps", choices=["primary", "cwd"], @@ -530,7 +560,7 @@ def arg_parser() -> argparse.ArgumentParser: parser.add_argument( "--enable-ext", action="store_true", - help="Enable loading and running 'cwltool:' extensions to the CWL standards.", + help="Enable loading and running `cwltool:` extensions to the CWL standards.", default=False, ) @@ -548,22 +578,6 @@ def arg_parser() -> argparse.ArgumentParser: help="Disable colored logging (default false)", ) - parser.add_argument( - "--default-container", - help="Specify a default software container to use for any " - "CommandLineTool without a DockerRequirement.", - ) - parser.add_argument( - "--no-match-user", - action="store_true", - help="Disable passing the current uid to `docker run --user`", - ) - parser.add_argument( - "--custom-net", - type=str, - help="Passed to `docker run` as the '--net' parameter when " - "NetworkAccess is true, which is its default setting.", - ) parser.add_argument( "--disable-validate", dest="do_validate", @@ -606,9 +620,9 @@ def arg_parser() -> argparse.ArgumentParser: parser.add_argument( "--on-error", - help="Desired workflow behavior when a step fails. One of 'stop' (do " - "not submit any more steps) or 'continue' (may submit other steps that " - "are not downstream from the error). Default is 'stop'.", + help="Desired workflow behavior when a step fails. One of `stop` (do " + "not submit any more steps) or `continue` (may submit other steps that " + "are not downstream from the error). Default is `stop`.", default="stop", choices=("stop", "continue"), ) @@ -636,21 +650,6 @@ def arg_parser() -> argparse.ArgumentParser: dest="relax_path_checks", ) - parser.add_argument( - "--force-docker-pull", - action="store_true", - default=False, - help="Pull latest software container image even if it is locally present", - dest="force_docker_pull", - ) - parser.add_argument( - "--no-read-only", - action="store_true", - default=False, - help="Do not set root directory in the container as read-only", - dest="no_read_only", - ) - parser.add_argument( "--overrides", type=str, @@ -658,7 +657,8 @@ def arg_parser() -> argparse.ArgumentParser: help="Read process requirement overrides from file.", ) - subgroup = parser.add_mutually_exclusive_group() + target_group = parser.add_argument_group(title="Target selection (optional)") + subgroup = target_group.add_mutually_exclusive_group() subgroup.add_argument( "--target", "-t", @@ -679,8 +679,8 @@ def arg_parser() -> argparse.ArgumentParser: default=None, help="Only executes the underlying Process (CommandLineTool, " "ExpressionTool, or sub-Workflow) for the given step in a workflow. " - "This will not include any step-level processing: 'scatter', 'when'; " - "and there will be no processing of step-level 'default', or 'valueFrom' " + "This will not include any step-level processing: `scatter`, `when`; " + "and there will be no processing of step-level `default`, or `valueFrom` " "input modifiers. However, requirements/hints from the step or parent " "workflow(s) will be inherited as usual." "The input object must match that Process's inputs.", @@ -714,11 +714,15 @@ def arg_parser() -> argparse.ArgumentParser: "formatted description of the required input values for the given " "`cwl_document`.", ) - + parser.add_argument( + "--generate-help-preview", + action=HelpPreviewAction, + path="help-preview.svg", # (optional) or "help-preview.html" or "help-preview.txt" + ) return parser -def get_default_args() -> Dict[str, Any]: +def get_default_args() -> dict[str, Any]: """Get default values of cwltool's command line options.""" ap = arg_parser() args = ap.parse_args([]) @@ -732,7 +736,7 @@ class FSAction(argparse.Action): def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, @@ -770,7 +774,7 @@ class FSAppendAction(argparse.Action): def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, @@ -827,7 +831,7 @@ class AppendAction(argparse.Action): def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, **kwargs: Any, @@ -859,13 +863,14 @@ def add_argument( toolparser: argparse.ArgumentParser, name: str, inptype: Any, - records: List[str], + records: list[str], description: str = "", default: Any = None, input_required: bool = True, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, base_uri: str = "", ) -> None: + description = rich.markup.escape(description) if len(name) == 1: flag = "-" else: @@ -888,9 +893,9 @@ def add_argument( return None ahelp = description.replace("%", "%%") - action: Optional[Union[Type[argparse.Action], str]] = None + action: Optional[Union[type[argparse.Action], str]] = None atype: Optional[Any] = None - typekw: Dict[str, Any] = {} + typekw: dict[str, Any] = {} if inptype == "File": action = FileAction @@ -962,8 +967,8 @@ def add_argument( def generate_parser( toolparser: argparse.ArgumentParser, tool: Process, - namemap: Dict[str, str], - records: List[str], + namemap: dict[str, str], + records: list[str], input_required: bool = True, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, base_uri: str = "", @@ -991,4 +996,10 @@ def generate_parser( base_uri, ) + toolparser.add_argument( + "--generate-help-preview", + action=HelpPreviewAction, + path="help-preview.svg", # (optional) or "help-preview.html" or "help-preview.txt" + ) + return toolparser diff --git a/cwltool/builder.py b/cwltool/builder.py index 2ba1e6543..e1de5b857 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -3,21 +3,9 @@ import copy import logging import math +from collections.abc import MutableMapping, MutableSequence from decimal import Decimal -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Type, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, Callable, Optional, Union, cast from cwl_utils import expression from cwl_utils.file_formats import check_format @@ -55,7 +43,7 @@ ) from .pathmapper import PathMapper -INPUT_OBJ_VOCAB: Dict[str, str] = { +INPUT_OBJ_VOCAB: dict[str, str] = { "Any": "https://w3id.org/cwl/salad#Any", "File": "https://w3id.org/cwl/cwl#File", "Directory": "https://w3id.org/cwl/cwl#Directory", @@ -107,16 +95,16 @@ class Builder(HasReqsHints): def __init__( self, job: CWLObjectType, - files: List[CWLObjectType], - bindings: List[CWLObjectType], + files: list[CWLObjectType], + bindings: list[CWLObjectType], schemaDefs: MutableMapping[str, CWLObjectType], names: Names, - requirements: List[CWLObjectType], - hints: List[CWLObjectType], - resources: Dict[str, Union[int, float]], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], + resources: dict[str, Union[int, float]], mutation_manager: Optional[MutationManager], formatgraph: Optional[Graph], - make_fs_access: Type[StdFsAccess], + make_fs_access: type[StdFsAccess], fs_access: StdFsAccess, job_script_provider: Optional[DependenciesConfiguration], timeout: float, @@ -172,7 +160,8 @@ def __init__( self.find_default_container: Optional[Callable[[], str]] = None self.container_engine = container_engine - def build_job_script(self, commands: List[str]) -> Optional[str]: + def build_job_script(self, commands: list[str]) -> Optional[str]: + """Use the job_script_provider to turn the commands into a job script.""" if self.job_script_provider is not None: return self.job_script_provider.build_job_script(self, commands) return None @@ -180,11 +169,11 @@ def build_job_script(self, commands: List[str]) -> Optional[str]: def bind_input( self, schema: CWLObjectType, - datum: Union[CWLObjectType, List[CWLObjectType]], + datum: Union[CWLObjectType, list[CWLObjectType]], discover_secondaryFiles: bool, - lead_pos: Optional[Union[int, List[int]]] = None, - tail_pos: Optional[Union[str, List[int]]] = None, - ) -> List[MutableMapping[str, Union[str, List[int]]]]: + lead_pos: Optional[Union[int, list[int]]] = None, + tail_pos: Optional[Union[str, list[int]]] = None, + ) -> list[MutableMapping[str, Union[str, list[int]]]]: """ Bind an input object to the command line. @@ -200,8 +189,8 @@ def bind_input( if lead_pos is None: lead_pos = [] - bindings: List[MutableMapping[str, Union[str, List[int]]]] = [] - binding: Union[MutableMapping[str, Union[str, List[int]]], CommentedMap] = {} + bindings: list[MutableMapping[str, Union[str, list[int]]]] = [] + binding: Union[MutableMapping[str, Union[str, list[int]]], CommentedMap] = {} value_from_expression = False if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping): binding = CommentedMap(schema["inputBinding"].items()) @@ -324,7 +313,7 @@ def bind_input( if schema["type"] == "record": datum = cast(CWLObjectType, datum) - for f in cast(List[CWLObjectType], schema["fields"]): + for f in cast(list[CWLObjectType], schema["fields"]): name = cast(str, f["name"]) if name in datum and datum[name] is not None: bindings.extend( @@ -372,7 +361,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: self.files.append(datum) loadContents_sourceline: Union[ - None, MutableMapping[str, Union[str, List[int]]], CWLObjectType + None, MutableMapping[str, Union[str, list[int]]], CWLObjectType ] = None if binding and binding.get("loadContents"): loadContents_sourceline = binding @@ -513,7 +502,7 @@ def addsf( if "format" in schema: eval_format: Any = self.do_eval(schema["format"]) if isinstance(eval_format, str): - evaluated_format: Union[str, List[str]] = eval_format + evaluated_format: Union[str, list[str]] = eval_format elif isinstance(eval_format, MutableSequence): for index, entry in enumerate(eval_format): message = None @@ -541,7 +530,7 @@ def addsf( raise SourceLine( schema["format"], index, WorkflowException, debug ).makeError(message) - evaluated_format = cast(List[str], eval_format) + evaluated_format = cast(list[str], eval_format) else: raise SourceLine(schema, "format", WorkflowException, debug).makeError( "An expression in the 'format' field must " @@ -586,8 +575,8 @@ def addsf( # Position to front of the sort key if binding: for bi in bindings: - bi["position"] = cast(List[int], binding["position"]) + cast( - List[int], bi["position"] + bi["position"] = cast(list[int], binding["position"]) + cast( + list[int], bi["position"] ) bindings.append(binding) @@ -618,7 +607,8 @@ def tostr(self, value: Union[MutableMapping[str, str], Any]) -> str: else: return str(value) - def generate_arg(self, binding: CWLObjectType) -> List[str]: + def generate_arg(self, binding: CWLObjectType) -> list[str]: + """Convert an input binding to a list of command line arguments.""" value = binding.get("datum") debug = _logger.isEnabledFor(logging.DEBUG) if "valueFrom" in binding: @@ -648,7 +638,7 @@ def generate_arg(self, binding: CWLObjectType) -> List[str]: argl = [itemSeparator.join([self.tostr(v) for v in value])] elif binding.get("valueFrom"): value = [self.tostr(v) for v in value] - return cast(List[str], ([prefix] if prefix else [])) + cast(List[str], value) + return cast(list[str], ([prefix] if prefix else [])) + cast(list[str], value) elif prefix and value: return [prefix] else: diff --git a/cwltool/checker.py b/cwltool/checker.py index 676245698..a3b8ba5df 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -1,19 +1,7 @@ """Static checking of CWL workflow connectivity.""" -from collections import namedtuple -from typing import ( - Any, - Dict, - Iterator, - List, - Literal, - MutableMapping, - MutableSequence, - Optional, - Sized, - Union, - cast, -) +from collections.abc import Iterator, MutableMapping, MutableSequence, Sized +from typing import Any, Literal, NamedTuple, Optional, Union, cast from schema_salad.exceptions import ValidationException from schema_salad.sourceline import SourceLine, bullets, strip_dup_lineno @@ -25,8 +13,7 @@ from .utils import CWLObjectType, CWLOutputType, SinkType, aslist -def _get_type(tp): - # type: (Any) -> Any +def _get_type(tp: Any) -> Any: if isinstance(tp, MutableMapping): if tp.get("type") not in ("array", "record", "enum"): return tp["type"] @@ -98,10 +85,10 @@ def can_assign_src_to_sink(src: SinkType, sink: Optional[SinkType], strict: bool if src["type"] == "record" and sink["type"] == "record": return _compare_records(src, sink, strict) if src["type"] == "File" and sink["type"] == "File": - for sinksf in cast(List[CWLObjectType], sink.get("secondaryFiles", [])): + for sinksf in cast(list[CWLObjectType], sink.get("secondaryFiles", [])): if not [ 1 - for srcsf in cast(List[CWLObjectType], src.get("secondaryFiles", [])) + for srcsf in cast(list[CWLObjectType], src.get("secondaryFiles", [])) if sinksf == srcsf ]: if strict: @@ -167,7 +154,8 @@ def _rec_fields(rec: MutableMapping[str, Any]) -> MutableMapping[str, Any]: return True -def missing_subset(fullset: List[Any], subset: List[Any]) -> List[Any]: +def missing_subset(fullset: list[Any], subset: list[Any]) -> list[Any]: + """Calculate the items missing from the fullset given the subset.""" missing = [] for i in subset: if i not in fullset: @@ -176,11 +164,11 @@ def missing_subset(fullset: List[Any], subset: List[Any]) -> List[Any]: def static_checker( - workflow_inputs: List[CWLObjectType], + workflow_inputs: list[CWLObjectType], workflow_outputs: MutableSequence[CWLObjectType], step_inputs: MutableSequence[CWLObjectType], - step_outputs: List[CWLObjectType], - param_to_step: Dict[str, CWLObjectType], + step_outputs: list[CWLObjectType], + param_to_step: dict[str, CWLObjectType], ) -> None: """ Check if all source and sink types of a workflow are compatible before run time. @@ -191,12 +179,12 @@ def static_checker( # sink parameters: step_inputs and workflow_outputs # make a dictionary of source parameters, indexed by the "id" field - src_dict: Dict[str, CWLObjectType] = {} + src_dict: dict[str, CWLObjectType] = {} for param in workflow_inputs + step_outputs: src_dict[cast(str, param["id"])] = param - step_inputs_val = check_all_types(src_dict, step_inputs, "source", param_to_step) - workflow_outputs_val = check_all_types( + step_inputs_val = _check_all_types(src_dict, step_inputs, "source", param_to_step) + workflow_outputs_val = _check_all_types( src_dict, workflow_outputs, "outputSource", param_to_step ) @@ -210,27 +198,34 @@ def static_checker( sink = warning.sink linkMerge = warning.linkMerge sinksf = sorted( - p["pattern"] for p in sink.get("secondaryFiles", []) if p.get("required", True) + cast(str, p["pattern"]) + for p in cast(MutableSequence[CWLObjectType], sink.get("secondaryFiles", [])) + if p.get("required", True) + ) + srcsf = sorted( + cast(str, p["pattern"]) + for p in cast(MutableSequence[CWLObjectType], src.get("secondaryFiles", [])) ) - srcsf = sorted(p["pattern"] for p in src.get("secondaryFiles", [])) # Every secondaryFile required by the sink, should be declared # by the source missing = missing_subset(srcsf, sinksf) + src_name = shortname(cast(str, src["id"])) + sink_id = cast(str, sink["id"]) + sink_name = shortname(sink_id) if missing: msg1 = "Parameter '{}' requires secondaryFiles {} but".format( - shortname(sink["id"]), + sink_name, missing, ) msg3 = SourceLine(src, "id").makeError( - "source '%s' does not provide those secondaryFiles." % (shortname(src["id"])) + "source '%s' does not provide those secondaryFiles." % (src_name) ) msg4 = SourceLine(src.get("_tool_entry", src), "secondaryFiles").makeError( "To resolve, add missing secondaryFiles patterns to definition of '%s' or" - % (shortname(src["id"])) + % (src_name) ) msg5 = SourceLine(sink.get("_tool_entry", sink), "secondaryFiles").makeError( - "mark missing secondaryFiles in definition of '%s' as optional." - % shortname(sink["id"]) + "mark missing secondaryFiles in definition of '%s' as optional." % (sink_name) ) msg = SourceLine(sink).makeError( "{}\n{}".format(msg1, bullets([msg3, msg4, msg5], " ")) @@ -240,13 +235,13 @@ def static_checker( msg = SourceLine(sink, "type").makeError( "'%s' is not an input parameter of %s, expected %s" % ( - shortname(sink["id"]), - param_to_step[sink["id"]]["run"], + sink_name, + param_to_step[sink_id]["run"], ", ".join( shortname(cast(str, s["id"])) for s in cast( - List[Dict[str, Union[str, bool]]], - param_to_step[sink["id"]]["inputs"], + list[dict[str, Union[str, bool]]], + param_to_step[sink_id]["inputs"], ) if not s.get("not_connected") ), @@ -258,12 +253,11 @@ def static_checker( msg = ( SourceLine(src, "type").makeError( "Source '%s' of type %s may be incompatible" - % (shortname(src["id"]), json_dumps(src["type"])) + % (src_name, json_dumps(src["type"])) ) + "\n" + SourceLine(sink, "type").makeError( - " with sink '%s' of type %s" - % (shortname(sink["id"]), json_dumps(sink["type"])) + " with sink '%s' of type %s" % (sink_name, json_dumps(sink["type"])) ) ) if linkMerge is not None: @@ -285,12 +279,12 @@ def static_checker( msg = ( SourceLine(src, "type").makeError( "Source '%s' of type %s is incompatible" - % (shortname(src["id"]), json_dumps(src["type"])) + % (shortname(cast(str, src["id"])), json_dumps(src["type"])) ) + "\n" + SourceLine(sink, "type").makeError( " with sink '{}' of type {}".format( - shortname(sink["id"]), json_dumps(sink["type"]) + shortname(cast(str, sink["id"])), json_dumps(sink["type"]) ) ) ) @@ -302,16 +296,17 @@ def static_checker( exception_msgs.append(msg) for sink in step_inputs: + sink_type = cast(Union[str, list[str], list[CWLObjectType], CWLObjectType], sink["type"]) if ( - "null" != sink["type"] - and "null" not in sink["type"] + "null" != sink_type + and "null" not in sink_type and "source" not in sink and "default" not in sink and "valueFrom" not in sink ): msg = SourceLine(sink).makeError( "Required parameter '%s' does not have source, default, or valueFrom expression" - % shortname(sink["id"]) + % shortname(cast(str, sink["id"])) ) exception_msgs.append(msg) @@ -324,15 +319,21 @@ def static_checker( raise ValidationException(all_exception_msg) -SrcSink = namedtuple("SrcSink", ["src", "sink", "linkMerge", "message"]) +class _SrcSink(NamedTuple): + """An error or warning message about a connection between two points of the workflow graph.""" + + src: CWLObjectType + sink: CWLObjectType + linkMerge: Optional[str] + message: Optional[str] -def check_all_types( - src_dict: Dict[str, CWLObjectType], +def _check_all_types( + src_dict: dict[str, CWLObjectType], sinks: MutableSequence[CWLObjectType], sourceField: Union[Literal["source"], Literal["outputSource"]], - param_to_step: Dict[str, CWLObjectType], -) -> Dict[str, List[SrcSink]]: + param_to_step: dict[str, CWLObjectType], +) -> dict[str, list[_SrcSink]]: """ Given a list of sinks, check if their types match with the types of their sources. @@ -340,7 +341,7 @@ def check_all_types( (from :py:func:`check_types`) :raises ValidationException: if a sourceField is missing """ - validation = {"warning": [], "exception": []} # type: Dict[str, List[SrcSink]] + validation: dict[str, list[_SrcSink]] = {"warning": [], "exception": []} for sink in sinks: if sourceField in sink: valueFrom = cast(Optional[str], sink.get("valueFrom")) @@ -351,23 +352,23 @@ def check_all_types( extra_message = "pickValue is: %s" % pickValue if isinstance(sink[sourceField], MutableSequence): - linkMerge = cast( + linkMerge: Optional[str] = cast( Optional[str], sink.get( "linkMerge", ("merge_nested" if len(cast(Sized, sink[sourceField])) > 1 else None), ), - ) # type: Optional[str] + ) if pickValue in ["first_non_null", "the_only_non_null"]: linkMerge = None - srcs_of_sink = [] # type: List[CWLObjectType] + srcs_of_sink: list[CWLObjectType] = [] for parm_id in cast(MutableSequence[str], sink[sourceField]): srcs_of_sink += [src_dict[parm_id]] if is_conditional_step(param_to_step, parm_id) and pickValue is None: validation["warning"].append( - SrcSink( + _SrcSink( src_dict[parm_id], sink, linkMerge, @@ -391,7 +392,7 @@ def check_all_types( if pickValue is not None: validation["warning"].append( - SrcSink( + _SrcSink( src_dict[parm_id], sink, linkMerge, @@ -404,13 +405,13 @@ def check_all_types( snk_typ = sink["type"] if "null" not in src_typ: - src_typ = ["null"] + cast(List[Any], src_typ) + src_typ = ["null"] + cast(list[Any], src_typ) if "null" not in cast( - Union[List[str], CWLObjectType], snk_typ + Union[list[str], CWLObjectType], snk_typ ): # Given our type names this works even if not a list validation["warning"].append( - SrcSink( + _SrcSink( src_dict[parm_id], sink, linkMerge, @@ -430,17 +431,17 @@ def check_all_types( check_result = check_types(src, sink, linkMerge, valueFrom) if check_result == "warning": validation["warning"].append( - SrcSink(src, sink, linkMerge, message=extra_message) + _SrcSink(src, sink, linkMerge, message=extra_message) ) elif check_result == "exception": validation["exception"].append( - SrcSink(src, sink, linkMerge, message=extra_message) + _SrcSink(src, sink, linkMerge, message=extra_message) ) return validation -def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: +def circular_dependency_checker(step_inputs: list[CWLObjectType]) -> None: """ Check if a workflow has circular dependency. @@ -448,8 +449,8 @@ def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: """ adjacency = get_dependency_tree(step_inputs) vertices = adjacency.keys() - processed: List[str] = [] - cycles: List[List[str]] = [] + processed: list[str] = [] + cycles: list[list[str]] = [] for vertex in vertices: if vertex not in processed: traversal_path = [vertex] @@ -461,7 +462,7 @@ def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: raise ValidationException(exception_msg) -def get_dependency_tree(step_inputs: List[CWLObjectType]) -> Dict[str, List[str]]: +def get_dependency_tree(step_inputs: list[CWLObjectType]) -> dict[str, list[str]]: """Get the dependency tree in the form of adjacency list.""" adjacency = {} # adjacency list of the dependency tree for step_input in step_inputs: @@ -482,10 +483,10 @@ def get_dependency_tree(step_inputs: List[CWLObjectType]) -> Dict[str, List[str] def processDFS( - adjacency: Dict[str, List[str]], - traversal_path: List[str], - processed: List[str], - cycles: List[List[str]], + adjacency: dict[str, list[str]], + traversal_path: list[str], + processed: list[str], + cycles: list[list[str]], ) -> None: """Perform depth first search.""" tip = traversal_path[-1] @@ -509,14 +510,15 @@ def get_step_id(field_id: str) -> str: return step_id -def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool: +def is_conditional_step(param_to_step: dict[str, CWLObjectType], parm_id: str) -> bool: + """Return True if the step given by the parm_id is a conditional step.""" if (source_step := param_to_step.get(parm_id)) is not None: if source_step.get("when") is not None: return True return False -def is_all_output_method_loop_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool: +def is_all_output_method_loop_step(param_to_step: dict[str, CWLObjectType], parm_id: str) -> bool: """Check if a step contains a `loop` directive with `all_iterations` outputMethod.""" source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id) if source_step is not None: diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 7a4e8ff71..cbff45bd0 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -7,31 +7,17 @@ import logging import os import re +import shlex import shutil import threading import urllib import urllib.parse +from collections.abc import Generator, Mapping, MutableMapping, MutableSequence from enum import Enum from functools import cmp_to_key, partial -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Generator, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Pattern, - Set, - TextIO, - Type, - Union, - cast, -) +from re import Pattern +from typing import TYPE_CHECKING, Any, Optional, TextIO, Union, cast -import shellescape from mypy_extensions import mypyc_attr from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import Schema @@ -163,8 +149,8 @@ def __init__( builder: Builder, script: str, output_callback: Optional[OutputCallbackType], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], outdir: Optional[str] = None, tmpdir: Optional[str] = None, ) -> None: @@ -241,7 +227,8 @@ def job( raise WorkflowException("Abstract operation cannot be executed.") -def remove_path(f): # type: (CWLObjectType) -> None +def remove_path(f: CWLObjectType) -> None: + """Remove any 'path' property, if present.""" if "path" in f: del f["path"] @@ -289,7 +276,7 @@ def revmap_file(builder: Builder, outdir: str, f: CWLObjectType) -> Optional[CWL ) revmap_f = builder.pathmapper.reversemap(path) - if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"): + if revmap_f and not builder.pathmapper.mapper(revmap_f[0]).type.startswith("Writable"): # type: ignore[union-attr] f["location"] = revmap_f[1] elif ( uripath == outdir @@ -334,7 +321,7 @@ def __init__( self.output_callback = output_callback self.cachebuilder = cachebuilder self.outdir = jobcache - self.prov_obj = None # type: Optional[ProvenanceProfile] + self.prov_obj: Optional[ProvenanceProfile] = None def run( self, @@ -392,7 +379,7 @@ def check_valid_locations(fs_access: StdFsAccess, ob: CWLObjectType) -> None: raise ValidationException("Does not exist or is not a Directory: '%s'" % location) -OutputPortsType = Dict[str, Optional[CWLOutputType]] +OutputPortsType = dict[str, Optional[CWLOutputType]] class ParameterOutputWorkflowException(WorkflowException): @@ -411,13 +398,14 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext """Initialize this CommandLineTool.""" super().__init__(toolpath_object, loadingContext) self.prov_obj = loadingContext.prov_obj - self.path_check_mode = ( + self.path_check_mode: PathCheckingMode = ( PathCheckingMode.RELAXED if loadingContext.relax_path_checks else PathCheckingMode.STRICT - ) # type: PathCheckingMode + ) - def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: + def make_job_runner(self, runtimeContext: RuntimeContext) -> type[JobBase]: + """Return the correct CommandLineJob class given the container settings.""" dockerReq, dockerRequired = self.get_requirement("DockerRequirement") mpiReq, mpiRequired = self.get_requirement(MPIRequirementName) @@ -443,28 +431,6 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: return SingularityCommandLineJob elif runtimeContext.user_space_docker_cmd: return UDockerCommandLineJob - if mpiReq is not None: - if mpiRequired: - if dockerRequired: - raise UnsupportedRequirement( - "No support for Docker and MPIRequirement both being required" - ) - else: - _logger.warning( - "MPI has been required while Docker is hinted, discarding Docker hint(s)" - ) - self.hints = [h for h in self.hints if h["class"] != "DockerRequirement"] - return CommandLineJob - else: - if dockerRequired: - _logger.warning( - "Docker has been required while MPI is hinted, discarding MPI hint(s)" - ) - self.hints = [h for h in self.hints if h["class"] != MPIRequirementName] - else: - raise UnsupportedRequirement( - "Both Docker and MPI have been hinted - don't know what to do" - ) if runtimeContext.podman: return PodmanCommandLineJob return DockerCommandLineJob @@ -477,7 +443,7 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: @staticmethod def make_path_mapper( - reffiles: List[CWLObjectType], + reffiles: list[CWLObjectType], stagedir: str, runtimeContext: RuntimeContext, separateDirs: bool, @@ -499,12 +465,19 @@ def updatePathmap(self, outdir: str, pathmap: PathMapper, fn: CWLObjectType) -> ("Writable" if fn.get("writable") else "") + cast(str, fn["class"]), False, ) - for sf in cast(List[CWLObjectType], fn.get("secondaryFiles", [])): + for sf in cast(list[CWLObjectType], fn.get("secondaryFiles", [])): self.updatePathmap(outdir, pathmap, sf) - for ls in cast(List[CWLObjectType], fn.get("listing", [])): + for ls in cast(list[CWLObjectType], fn.get("listing", [])): self.updatePathmap(os.path.join(outdir, cast(str, fn["basename"])), pathmap, ls) - def _initialworkdir(self, j: JobBase, builder: Builder) -> None: + def _initialworkdir(self, j: Optional[JobBase], builder: Builder) -> None: + """ + Test and initialize the working directory. + + :param j: A :py:class:`~cwltool.job.CommandLineJob` or a + specialized container-based job. + If 'None', then only tests will be performed, no setup. + """ initialWorkdir, _ = self.get_requirement("InitialWorkDirRequirement") if initialWorkdir is None: return @@ -517,7 +490,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: cwl_version ) < ORDERED_VERSIONS.index("v1.1.0-dev1") - ls = [] # type: List[CWLObjectType] + ls: list[CWLObjectType] = [] if isinstance(initialWorkdir["listing"], str): # "listing" is just a string (must be an expression) so # just evaluate it and use the result as if it was in @@ -587,7 +560,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError( message ) - ls = cast(List[CWLObjectType], ls_evaluated) + ls = cast(list[CWLObjectType], ls_evaluated) else: # "listing" is an array of either expressions or Dirent so # evaluate each item @@ -634,10 +607,10 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: for e in entry: ec = cast(CWLObjectType, e) ec["writable"] = t.get("writable", False) - ls.extend(cast(List[CWLObjectType], entry)) + ls.extend(cast(list[CWLObjectType], entry)) continue - et = {} # type: CWLObjectType + et: CWLObjectType = {} if isinstance(entry, Mapping) and entry.get("class") in ( "File", "Directory", @@ -686,7 +659,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: if not initwd_item: continue if isinstance(initwd_item, MutableSequence): - ls.extend(cast(List[CWLObjectType], initwd_item)) + ls.extend(cast(list[CWLObjectType], initwd_item)) else: ls.append(cast(CWLObjectType, initwd_item)) @@ -772,6 +745,9 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: "is in 'requirements'." ) + if j is None: + return # Only testing + with SourceLine(initialWorkdir, "listing", WorkflowException, debug): j.generatefiles["listing"] = ls for entry in ls: @@ -819,10 +795,10 @@ def job( cachecontext.tmpdir = "/tmp" # nosec cachecontext.stagedir = "/stage" cachebuilder = self._init_job(job_order, cachecontext) - cachebuilder.pathmapper = PathMapper( + cachebuilder.pathmapper = self.make_path_mapper( cachebuilder.files, - runtimeContext.basedir, cachebuilder.stagedir, + runtimeContext, separateDirs=False, ) _check_adjust = partial(check_adjust, self.path_check_mode.value, cachebuilder) @@ -836,6 +812,7 @@ def job( _check_adjust, ) visit_class([cachebuilder.files, cachebuilder.bindings], ("File"), _checksum) + self._initialworkdir(None, cachebuilder) # test the initial working directory cmdline = flatten(list(map(cachebuilder.generate_arg, cachebuilder.bindings))) docker_req, _ = self.get_requirement("DockerRequirement") @@ -850,9 +827,9 @@ def job( cmdline = ["docker", "run", dockerimg] + cmdline # not really run using docker, just for hashing purposes - keydict = { + keydict: dict[str, Union[MutableSequence[Union[str, int]], CWLObjectType]] = { "cmdline": cmdline - } # type: Dict[str, Union[MutableSequence[Union[str, int]], CWLObjectType]] + } for shortcut in ["stdin", "stdout", "stderr"]: if shortcut in self.tool: @@ -1071,8 +1048,8 @@ def update_status_output_callback( j.inplace_update = cast(bool, inplaceUpdateReq["inplaceUpdate"]) normalizeFilesDirs(j.generatefiles) - readers = {} # type: Dict[str, CWLObjectType] - muts = set() # type: Set[str] + readers: dict[str, CWLObjectType] = {} + muts: set[str] = set() if builder.mutation_manager is not None: @@ -1103,7 +1080,7 @@ def register_reader(f: CWLObjectType) -> None: timelimit, _ = self.get_requirement("ToolTimeLimit") if timelimit is not None: with SourceLine(timelimit, "timelimit", ValidationException, debug): - limit_field = cast(Dict[str, Union[str, int]], timelimit)["timelimit"] + limit_field = cast(dict[str, Union[str, int]], timelimit)["timelimit"] if isinstance(limit_field, str): timelimit_eval = builder.do_eval(limit_field) if timelimit_eval and not isinstance(timelimit_eval, int): @@ -1142,7 +1119,7 @@ def register_reader(f: CWLObjectType) -> None: required_env = {} evr, _ = self.get_requirement("EnvVarRequirement") if evr is not None: - for eindex, t3 in enumerate(cast(List[Dict[str, str]], evr["envDef"])): + for eindex, t3 in enumerate(cast(list[dict[str, str]], evr["envDef"])): env_value_field = t3["envValue"] if "${" in env_value_field or "$(" in env_value_field: env_value_eval = builder.do_eval(env_value_field) @@ -1160,11 +1137,11 @@ def register_reader(f: CWLObjectType) -> None: shellcmd, _ = self.get_requirement("ShellCommandRequirement") if shellcmd is not None: - cmd = [] # type: List[str] + cmd: list[str] = [] for b in builder.bindings: arg = builder.generate_arg(b) if b.get("shellQuote", True): - arg = [shellescape.quote(a) for a in aslist(arg)] + arg = [shlex.quote(a) for a in aslist(arg)] cmd.extend(aslist(arg)) j.command_line = ["/bin/sh", "-c", " ".join(cmd)] else: @@ -1201,7 +1178,7 @@ def register_reader(f: CWLObjectType) -> None: def collect_output_ports( self, - ports: Union[CommentedSeq, Set[CWLObjectType]], + ports: Union[CommentedSeq, set[CWLObjectType]], builder: Builder, outdir: str, rcode: int, @@ -1209,7 +1186,7 @@ def collect_output_ports( jobname: str = "", readers: Optional[MutableMapping[str, CWLObjectType]] = None, ) -> OutputPortsType: - ret = {} # type: OutputPortsType + ret: OutputPortsType = {} debug = _logger.isEnabledFor(logging.DEBUG) cwl_version = self.metadata.get(ORIGINAL_CWLVERSION, None) if cwl_version != "v1.0": @@ -1284,16 +1261,16 @@ def collect_output( fs_access: StdFsAccess, compute_checksum: bool = True, ) -> Optional[CWLOutputType]: - r = [] # type: List[CWLOutputType] + r: list[CWLOutputType] = [] empty_and_optional = False debug = _logger.isEnabledFor(logging.DEBUG) result: Optional[CWLOutputType] = None if "outputBinding" in schema: binding = cast( - MutableMapping[str, Union[bool, str, List[str]]], + MutableMapping[str, Union[bool, str, list[str]]], schema["outputBinding"], ) - globpatterns = [] # type: List[str] + globpatterns: list[str] = [] revmap = partial(revmap_file, builder, outdir) @@ -1354,12 +1331,12 @@ def collect_output( ] ) except OSError as e: - _logger.warning(str(e)) + _logger.warning(str(e), exc_info=builder.debug) except Exception: _logger.error("Unexpected error from fs_access", exc_info=True) raise - for files in cast(List[Dict[str, Optional[CWLOutputType]]], r): + for files in cast(list[dict[str, Optional[CWLOutputType]]], r): rfile = files.copy() revmap(rfile) if files["class"] == "Directory": @@ -1515,7 +1492,7 @@ def collect_output( and schema["type"]["type"] == "record" ): out = {} - for field in cast(List[CWLObjectType], schema["type"]["fields"]): + for field in cast(list[CWLObjectType], schema["type"]["fields"]): out[shortname(cast(str, field["name"]))] = self.collect_output( field, builder, outdir, fs_access, compute_checksum=compute_checksum ) diff --git a/cwltool/context.py b/cwltool/context.py index 1e82ecc4a..bb281fd88 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -5,20 +5,8 @@ import shutil import tempfile import threading -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Literal, - Optional, - TextIO, - Tuple, - Union, -) +from collections.abc import Iterable +from typing import IO, TYPE_CHECKING, Any, Callable, Literal, Optional, TextIO, Union from ruamel.yaml.comments import CommentedMap from schema_salad.avro.schema import Names @@ -31,6 +19,7 @@ from .utils import DEFAULT_TMP_PREFIX, CWLObjectType, HasReqsHints, ResolverType if TYPE_CHECKING: + from _typeshed import SupportsWrite from cwl_utils.parser.cwl_v1_2 import LoadingOptions from .builder import Builder @@ -45,7 +34,7 @@ class ContextBase: """Shared kwargs based initializer for :py:class:`RuntimeContext` and :py:class:`LoadingContext`.""" - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: """Initialize.""" if kwargs: for k, v in kwargs.items(): @@ -86,13 +75,13 @@ def set_log_dir(outdir: str, log_dir: str, subdir_name: str) -> str: class LoadingContext(ContextBase): - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: """Initialize the LoadingContext from the kwargs.""" self.debug: bool = False self.metadata: CWLObjectType = {} - self.requirements: Optional[List[CWLObjectType]] = None - self.hints: Optional[List[CWLObjectType]] = None - self.overrides_list: List[CWLObjectType] = [] + self.requirements: Optional[list[CWLObjectType]] = None + self.hints: Optional[list[CWLObjectType]] = None + self.overrides_list: list[CWLObjectType] = [] self.loader: Optional[Loader] = None self.avsc_names: Optional[Names] = None self.disable_js_validation: bool = False @@ -116,7 +105,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.singularity: bool = False self.podman: bool = False self.eval_timeout: float = 60 - self.codegen_idx: Dict[str, Tuple[Any, "LoadingOptions"]] = {} + self.codegen_idx: dict[str, tuple[Any, "LoadingOptions"]] = {} self.fast_parser = False self.skip_resolve_all = False self.skip_schemas = False @@ -135,11 +124,11 @@ class RuntimeContext(ContextBase): tmp_outdir_prefix: str = "" stagedir: str = "" - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: """Initialize the RuntimeContext from the kwargs.""" select_resources_callable = Callable[ - [Dict[str, Union[int, float]], RuntimeContext], - Dict[str, Union[int, float]], + [dict[str, Union[int, float]], RuntimeContext], + dict[str, Union[int, float]], ] self.user_space_docker_cmd: Optional[str] = None self.secret_store: Optional["SecretStore"] = None @@ -194,12 +183,14 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.orcid: str = "" self.cwl_full_name: str = "" self.process_run_id: Optional[str] = None + self.prov_host: bool = False + self.prov_user: bool = False self.prov_obj: Optional[ProvenanceProfile] = None self.mpi_config: MpiConfig = MpiConfig() self.default_stdout: Optional[Union[IO[bytes], TextIO]] = None self.default_stderr: Optional[Union[IO[bytes], TextIO]] = None self.validate_only: bool = False - self.validate_stdout: Optional[Union[IO[bytes], TextIO, IO[str]]] = None + self.validate_stdout: Optional["SupportsWrite[str]"] = None super().__init__(kwargs) if self.tmp_outdir_prefix == "": self.tmp_outdir_prefix = self.tmpdir_prefix diff --git a/cwltool/cuda.py b/cwltool/cuda.py index 719bfd867..d607b83da 100644 --- a/cwltool/cuda.py +++ b/cwltool/cuda.py @@ -2,18 +2,18 @@ import subprocess # nosec import xml.dom.minidom # nosec -from typing import Tuple +from typing import Union from .loghandler import _logger from .utils import CWLObjectType -def cuda_version_and_device_count() -> Tuple[str, int]: +def cuda_version_and_device_count() -> tuple[str, int]: """Determine the CUDA version and number of attached CUDA GPUs.""" try: - out = subprocess.check_output(["nvidia-smi", "-q", "-x"]) # nosec + out: Union[str, bytes] = subprocess.check_output(["nvidia-smi", "-q", "-x"]) # nosec except Exception as e: - _logger.warning("Error checking CUDA version with nvidia-smi: %s", e) + _logger.warning("Error checking CUDA version with nvidia-smi: %s", e, exc_info=e) return ("", 0) dm = xml.dom.minidom.parseString(out) # nosec @@ -63,5 +63,5 @@ def cuda_check(cuda_req: CWLObjectType, requestCount: int) -> int: return 0 return requestCount except Exception as e: - _logger.warning("Error checking CUDA requirements: %s", e) + _logger.warning("Error checking CUDA requirements: %s", e, exc_info=e) return 0 diff --git a/cwltool/cwlprov/__init__.py b/cwltool/cwlprov/__init__.py index b8ff8d14d..a09a57c34 100644 --- a/cwltool/cwlprov/__init__.py +++ b/cwltool/cwlprov/__init__.py @@ -6,10 +6,10 @@ import re import uuid from getpass import getuser -from typing import IO, Any, Callable, Dict, List, Optional, Tuple, TypedDict, Union +from typing import IO, Any, Callable, Optional, TypedDict, Union -def _whoami() -> Tuple[str, str]: +def _whoami() -> tuple[str, str]: """Return the current operating system account as (username, fullname).""" username = getuser() try: @@ -106,8 +106,8 @@ def _valid_orcid(orcid: Optional[str]) -> str: { "uri": str, "about": str, - "content": Optional[Union[str, List[str]]], - "oa:motivatedBy": Dict[str, str], + "content": Optional[Union[str, list[str]]], + "oa:motivatedBy": dict[str, str], }, ) @@ -116,11 +116,11 @@ class Aggregate(TypedDict, total=False): """RO Aggregate class.""" uri: Optional[str] - bundledAs: Optional[Dict[str, Any]] + bundledAs: Optional[dict[str, Any]] mediatype: Optional[str] - conformsTo: Optional[Union[str, List[str]]] + conformsTo: Optional[Union[str, list[str]]] createdOn: Optional[str] - createdBy: Optional[Dict[str, str]] + createdBy: Optional[dict[str, str]] # Aggregate.bundledAs is actually type Aggregate, but cyclic definitions are not supported diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index ce8d63ad4..e2208378f 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -3,22 +3,10 @@ import logging import urllib import uuid +from collections.abc import MutableMapping, MutableSequence, Sequence from io import BytesIO from pathlib import PurePath, PurePosixPath -from socket import getfqdn -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Sequence, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from prov.identifier import Identifier, QualifiedName from prov.model import PROV, PROV_LABEL, PROV_TYPE, PROV_VALUE, ProvDocument, ProvEntity @@ -35,12 +23,10 @@ ACCOUNT_UUID, CWLPROV, ENCODING, - FOAF, METADATA, ORE, PROVENANCE, RO, - SCHEMA, SHA1, SHA256, TEXT_PLAIN, @@ -117,27 +103,8 @@ def __str__(self) -> str: """Represent this Provenvance profile as a string.""" return f"ProvenanceProfile <{self.workflow_run_uri}> in <{self.research_object}>" - def generate_prov_doc(self) -> Tuple[str, ProvDocument]: + def generate_prov_doc(self) -> tuple[str, ProvDocument]: """Add basic namespaces.""" - - def host_provenance(document: ProvDocument) -> None: - """Record host provenance.""" - document.add_namespace(CWLPROV) - document.add_namespace(UUID) - document.add_namespace(FOAF) - - hostname = getfqdn() - # won't have a foaf:accountServiceHomepage for unix hosts, but - # we can at least provide hostname - document.agent( - ACCOUNT_UUID, - { - PROV_TYPE: FOAF["OnlineAccount"], - "prov:location": hostname, - CWLPROV["hostname"]: hostname, - }, - ) - self.cwltool_version = f"cwltool {versionstring().split()[-1]}" self.document.add_namespace("wfprov", "http://purl.org/wf4ever/wfprov#") # document.add_namespace('prov', 'http://www.w3.org/ns/prov#') @@ -176,25 +143,10 @@ def host_provenance(document: ProvDocument) -> None: # .. but we always know cwltool was launched (directly or indirectly) # by a user account, as cwltool is a command line tool account = self.document.agent(ACCOUNT_UUID) - if self.orcid or self.full_name: - person: Dict[Union[str, Identifier], Any] = { - PROV_TYPE: PROV["Person"], - "prov:type": SCHEMA["Person"], - } - if self.full_name: - person["prov:label"] = self.full_name - person["foaf:name"] = self.full_name - person["schema:name"] = self.full_name - else: - # TODO: Look up name from ORCID API? - pass - agent = self.document.agent(self.orcid or uuid.uuid4().urn, person) - self.document.actedOnBehalfOf(account, agent) - else: - if self.host_provenance: - host_provenance(self.document) - if self.user_provenance: - self.research_object.user_provenance(self.document) + if self.host_provenance: + self.research_object.host_provenance(self.document) + if self.user_provenance or self.orcid or self.full_name: + self.research_object.user_provenance(self.document) # The execution of cwltool wfengine = self.document.agent( self.engine_uuid, @@ -291,7 +243,8 @@ def record_process_end( self.generate_output_prov(outputs, process_run_id, process_name) self.document.wasEndedBy(process_run_id, None, self.workflow_run_uri, when) - def declare_file(self, value: CWLObjectType) -> Tuple[ProvEntity, ProvEntity, str]: + def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, str]: + """Construct a FileEntity for the given CWL File object.""" if value["class"] != "File": raise ValueError("Must have class:File: %s" % value) # Need to determine file hash aka RO filename @@ -399,10 +352,10 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity: # dir_bundle.identifier, {PROV["type"]: ORE["ResourceMap"], # ORE["describes"]: coll_b.identifier}) - coll_attribs: List[Tuple[Union[str, Identifier], Any]] = [ + coll_attribs: list[tuple[Union[str, Identifier], Any]] = [ (ORE["isDescribedBy"], dir_bundle.identifier) ] - coll_b_attribs: List[Tuple[Union[str, Identifier], Any]] = [] + coll_b_attribs: list[tuple[Union[str, Identifier], Any]] = [] # FIXME: .listing might not be populated yet - hopefully # a later call to this method will sort that @@ -469,7 +422,7 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity: self.research_object.add_uri(coll.identifier.uri) return coll - def declare_string(self, value: str) -> Tuple[ProvEntity, str]: + def declare_string(self, value: str) -> tuple[ProvEntity, str]: """Save as string in UTF-8.""" byte_s = BytesIO(str(value).encode(ENCODING)) data_file = self.research_object.add_data_file(byte_s, content_type=TEXT_PLAIN) @@ -518,7 +471,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: # Already processed this value, but it might not be in this PROV entities = self.document.get_record(value["@id"]) if entities: - return cast(List[ProvEntity], entities)[0] + return cast(list[ProvEntity], entities)[0] # else, unknown in PROV, re-add below as if it's fresh # Base case - we found a File we need to update @@ -549,7 +502,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: coll.add_asserted_type(CWLPROV[value["class"]]) # Let's iterate and recurse - coll_attribs: List[Tuple[Union[str, Identifier], Any]] = [] + coll_attribs: list[tuple[Union[str, Identifier], Any]] = [] for key, val in value.items(): v_ent = self.declare_artefact(val) self.document.membership(coll, v_ent) @@ -593,7 +546,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: # FIXME: list value does not support adding "@id" return coll except TypeError: - _logger.warning("Unrecognized type %s of %r", type(value), value) + _logger.warning("Unrecognized type %s of %r", type(value), value, exc_info=True) # Let's just fall back to Python repr() entity = self.document.entity(uuid.uuid4().urn, {PROV_LABEL: repr(value)}) self.research_object.add_uri(entity.identifier.uri) @@ -601,7 +554,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: def used_artefacts( self, - job_order: Union[CWLObjectType, List[CWLObjectType]], + job_order: Union[CWLObjectType, list[CWLObjectType]], process_run_id: str, name: Optional[str] = None, ) -> None: @@ -704,7 +657,7 @@ def activity_has_provenance(self, activity: str, prov_ids: Sequence[Identifier]) """Add http://www.w3.org/TR/prov-aq/ relations to nested PROV files.""" # NOTE: The below will only work if the corresponding metadata/provenance arcp URI # is a pre-registered namespace in the PROV Document - attribs: List[Tuple[Union[str, Identifier], Any]] = [ + attribs: list[tuple[Union[str, Identifier], Any]] = [ (PROV["has_provenance"], prov_id) for prov_id in prov_ids ] self.document.activity(activity, other_attributes=attribs) @@ -713,7 +666,7 @@ def activity_has_provenance(self, activity: str, prov_ids: Sequence[Identifier]) uris = [i.uri for i in prov_ids] self.research_object.add_annotation(activity, uris, PROV["has_provenance"].uri) - def finalize_prov_profile(self, name: Optional[str]) -> List[QualifiedName]: + def finalize_prov_profile(self, name: Optional[str]) -> list[QualifiedName]: """Transfer the provenance related files to the RO.""" # NOTE: Relative posix path if name is None: diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index 7c6eaf5d6..be10d3d64 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -7,23 +7,13 @@ import tempfile import urllib import uuid +from collections.abc import MutableMapping, MutableSequence from pathlib import Path, PurePosixPath -from typing import ( - IO, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from socket import getfqdn +from typing import IO, TYPE_CHECKING, Any, Optional, Union, cast import prov.model as provM -from prov.model import PROV, ProvDocument +from prov.model import ProvDocument from ..loghandler import _logger from ..stdfsaccess import StdFsAccess @@ -38,6 +28,7 @@ from . import Aggregate, Annotation, AuthoredBy, _valid_orcid, _whoami, checksum_copy from .provenance_constants import ( ACCOUNT_UUID, + CWLPROV, CWLPROV_VERSION, DATA, ENCODING, @@ -46,6 +37,7 @@ METADATA, ORCID, PROVENANCE, + SCHEMA, SHA1, SHA256, SHA512, @@ -57,6 +49,9 @@ Hasher, ) +if TYPE_CHECKING: + from .provenance_profile import ProvenanceProfile # pylint: disable=unused-import + class ResearchObject: """CWLProv Research Object.""" @@ -75,12 +70,12 @@ def __init__( self.folder = create_tmp_dir(temp_prefix_ro) self.closed = False # map of filename "data/de/alsdklkas": 12398123 bytes - self.bagged_size: Dict[str, int] = {} - self.tagfiles: Set[str] = set() - self._file_provenance: Dict[str, Aggregate] = {} - self._external_aggregates: List[Aggregate] = [] - self.annotations: List[Annotation] = [] - self._content_types: Dict[str, str] = {} + self.bagged_size: dict[str, int] = {} + self.tagfiles: set[str] = set() + self._file_provenance: dict[str, Aggregate] = {} + self._external_aggregates: list[Aggregate] = [] + self.annotations: list[Annotation] = [] + self._content_types: dict[str, str] = {} self.fsaccess = fsaccess # These should be replaced by generate_prov_doc when workflow/run IDs are known: self.engine_uuid = f"urn:uuid:{uuid.uuid4()}" @@ -93,6 +88,34 @@ def __init__( self._initialize() _logger.debug("[provenance] Temporary research object: %s", self.folder) + def initialize_provenance( + self, + full_name: str, + host_provenance: bool, + user_provenance: bool, + orcid: str, + fsaccess: StdFsAccess, + run_uuid: Optional[uuid.UUID] = None, + ) -> "ProvenanceProfile": + """ + Provide a provenance profile initialization hook function. + + Allows overriding the default strategy to define the + provenance profile concepts and associations to extend + details as needed. + """ + from .provenance_profile import ProvenanceProfile + + return ProvenanceProfile( + research_object=self, + full_name=full_name, + host_provenance=host_provenance, + user_provenance=user_provenance, + orcid=orcid, + fsaccess=fsaccess, + run_uuid=run_uuid, + ) + def self_check(self) -> None: """Raise ValueError if this RO is closed.""" if self.closed: @@ -128,10 +151,22 @@ def _initialize_bagit(self) -> None: bag_it_file.write("BagIt-Version: 0.97\n") bag_it_file.write(f"Tag-File-Character-Encoding: {ENCODING}\n") + def resolve_user(self) -> tuple[str, str]: + """ + Provide a user provenance hook function. + + Allows overriding the default strategy to retrieve user provenance + in case the calling code can provide a better resolution. + The function must return a tuple of the (username, fullname) + that identifies the user. This user will be applied on top + to any provided ORCID or fullname by agent association. + """ + return _whoami() + def user_provenance(self, document: ProvDocument) -> None: """Add the user provenance.""" self.self_check() - (username, fullname) = _whoami() + (username, fullname) = self.resolve_user() if not self.full_name: self.full_name = fullname @@ -143,19 +178,21 @@ def user_provenance(self, document: ProvDocument) -> None: ACCOUNT_UUID, { provM.PROV_TYPE: FOAF["OnlineAccount"], - "prov:label": username, + provM.PROV_LABEL: username, FOAF["accountName"]: username, }, ) user = document.agent( self.orcid or USER_UUID, - { - provM.PROV_TYPE: PROV["Person"], - "prov:label": self.full_name, - FOAF["name"]: self.full_name, - FOAF["account"]: account, - }, + [ + (provM.PROV_TYPE, SCHEMA["Person"]), + (provM.PROV_TYPE, provM.PROV["Person"]), + (provM.PROV_LABEL, self.full_name), + (FOAF["name"], self.full_name), + (FOAF["account"], account), + (SCHEMA["name"], self.full_name), + ], ) # cwltool may be started on the shell (directly by user), # by shell script (indirectly by user) @@ -167,6 +204,35 @@ def user_provenance(self, document: ProvDocument) -> None: # get their name wrong!) document.actedOnBehalfOf(account, user) + def resolve_host(self) -> tuple[str, str]: + """ + Provide a host provenance hook function. + + Allows overriding the default strategy to retrieve host provenance + in case the calling code can provide a better resolution. + The function must return a tuple of the (fqdn, uri) that identifies the host. + """ + fqdn = getfqdn() + return fqdn, fqdn # allow for (fqdn, uri) to be distinct, but the same by default + + def host_provenance(self, document: ProvDocument) -> None: + """Record host provenance.""" + document.add_namespace(CWLPROV) + document.add_namespace(UUID) + document.add_namespace(FOAF) + + hostname, uri = self.resolve_host() + # won't have a foaf:accountServiceHomepage for unix hosts, but + # we can at least provide hostname + document.agent( + ACCOUNT_UUID, + { + provM.PROV_TYPE: FOAF["OnlineAccount"], + provM.PROV_LOCATION: uri, + CWLPROV["hostname"]: hostname, + }, + ) + def add_tagfile(self, path: str, timestamp: Optional[datetime.datetime] = None) -> None: """Add tag files to our research object.""" self.self_check() @@ -202,14 +268,14 @@ def add_tagfile(self, path: str, timestamp: Optional[datetime.datetime] = None) "conformsTo": None, } - def _ro_aggregates(self) -> List[Aggregate]: + def _ro_aggregates(self) -> list[Aggregate]: """Gather dictionary of files to be added to the manifest.""" def guess_mediatype( rel_path: str, - ) -> Tuple[Optional[str], Optional[Union[str, List[str]]]]: + ) -> tuple[Optional[str], Optional[Union[str, list[str]]]]: """Return the mediatypes.""" - media_types: Dict[Union[str, None], str] = { + media_types: dict[Union[str, None], str] = { # Adapted from # https://w3id.org/bundle/2014-11-05/#media-types "txt": TEXT_PLAIN, @@ -223,12 +289,12 @@ def guess_mediatype( "provn": 'text/provenance-notation; charset="UTF-8"', "nt": "application/n-triples", } - conforms_to: Dict[Union[str, None], str] = { + conforms_to: dict[Union[str, None], str] = { "provn": "http://www.w3.org/TR/2013/REC-prov-n-20130430/", "cwl": "https://w3id.org/cwl/", } - prov_conforms_to: Dict[str, str] = { + prov_conforms_to: dict[str, str] = { "provn": "http://www.w3.org/TR/2013/REC-prov-n-20130430/", "rdf": "http://www.w3.org/TR/2013/REC-prov-o-20130430/", "ttl": "http://www.w3.org/TR/2013/REC-prov-o-20130430/", @@ -244,7 +310,7 @@ def guess_mediatype( extension = None mediatype: Optional[str] = media_types.get(extension, None) - conformsTo: Optional[Union[str, List[str]]] = conforms_to.get(extension, None) + conformsTo: Optional[Union[str, list[str]]] = conforms_to.get(extension, None) # TODO: Open CWL file to read its declared "cwlVersion", e.g. # cwlVersion = "v1.0" @@ -261,7 +327,7 @@ def guess_mediatype( conformsTo = prov_conforms_to[extension] return (mediatype, conformsTo) - aggregates: List[Aggregate] = [] + aggregates: list[Aggregate] = [] for path in self.bagged_size.keys(): temp_path = PurePosixPath(path) folder = temp_path.parent @@ -291,7 +357,7 @@ def guess_mediatype( bundledAs.update(self._file_provenance[path]) else: aggregate_dict["bundledAs"] = cast( - Optional[Dict[str, Any]], self._file_provenance[path] + Optional[dict[str, Any]], self._file_provenance[path] ) else: # Probably made outside wf run, part of job object? @@ -343,7 +409,7 @@ def add_uri(self, uri: str, timestamp: Optional[datetime.datetime] = None) -> Ag return aggr def add_annotation( - self, about: str, content: List[str], motivated_by: str = "oa:describing" + self, about: str, content: list[str], motivated_by: str = "oa:describing" ) -> str: """Cheap URI relativize for current directory and /.""" self.self_check() @@ -359,9 +425,9 @@ def add_annotation( self.annotations.append(ann) return uri - def _ro_annotations(self) -> List[Annotation]: + def _ro_annotations(self) -> list[Annotation]: """Append base RO and provenance annotations to the list of annotations.""" - annotations: List[Annotation] = [] + annotations: list[Annotation] = [] annotations.append( { "uri": uuid.uuid4().urn, @@ -511,7 +577,7 @@ def add_data_file( def _self_made( self, timestamp: Optional[datetime.datetime] = None - ) -> Tuple[str, Dict[str, str]]: # createdOn, createdBy + ) -> tuple[str, dict[str, str]]: # createdOn, createdBy if timestamp is None: timestamp = datetime.datetime.now() return ( @@ -519,7 +585,7 @@ def _self_made( {"uri": self.engine_uuid, "name": self.cwltool_version}, ) - def add_to_manifest(self, rel_path: str, checksums: Dict[str, str]) -> None: + def add_to_manifest(self, rel_path: str, checksums: dict[str, str]) -> None: """Add files to the research object manifest.""" self.self_check() if PurePosixPath(rel_path).is_absolute(): @@ -602,7 +668,7 @@ def _relativise_files( del structure["path"] if structure.get("class") == "Directory": - # TODO: Generate anonymoys Directory with a "listing" + # TODO: Generate anonymous Directory with a "listing" # pointing to the hashed files del structure["location"] diff --git a/cwltool/cwlprov/writablebagfile.py b/cwltool/cwlprov/writablebagfile.py index d5ff3c731..06d7d0bf7 100644 --- a/cwltool/cwlprov/writablebagfile.py +++ b/cwltool/cwlprov/writablebagfile.py @@ -8,10 +8,11 @@ import uuid from array import array from collections import OrderedDict +from collections.abc import MutableMapping from io import FileIO, TextIOWrapper from mmap import mmap from pathlib import Path, PurePosixPath -from typing import Any, BinaryIO, Dict, MutableMapping, Optional, Union, cast +from typing import Any, BinaryIO, Optional, Union, cast from schema_salad.utils import json_dumps @@ -246,7 +247,7 @@ def create_job( relativised_input_objecttemp: CWLObjectType = {} research_object._relativise_files(copied) - def jdefault(o: Any) -> Dict[Any, Any]: + def jdefault(o: Any) -> dict[Any, Any]: return dict(o) if is_output: diff --git a/cwltool/cwlrdf.py b/cwltool/cwlrdf.py index dbe9e2f97..126f0c780 100644 --- a/cwltool/cwlrdf.py +++ b/cwltool/cwlrdf.py @@ -1,6 +1,7 @@ import urllib from codecs import StreamWriter -from typing import IO, Any, Dict, Iterator, Optional, TextIO, Union, cast +from collections.abc import Iterator +from typing import IO, Any, Optional, TextIO, Union, cast from rdflib import Graph from rdflib.query import ResultRow @@ -117,7 +118,7 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: - dotname: Dict[str, str] = {} + dotname: dict[str, str] = {} clusternode = {} stdout.write("compound=true\n") diff --git a/cwltool/cwlviewer.py b/cwltool/cwlviewer.py index e544a568e..36166c485 100644 --- a/cwltool/cwlviewer.py +++ b/cwltool/cwlviewer.py @@ -1,17 +1,28 @@ """Visualize a CWL workflow.""" -from pathlib import Path -from typing import Iterator, List, cast +from collections.abc import Iterator +from importlib.resources import files +from typing import cast from urllib.parse import urlparse import pydot import rdflib -_queries_dir = (Path(__file__).parent / "rdfqueries").resolve() -_get_inner_edges_query_path = _queries_dir / "get_inner_edges.sparql" -_get_input_edges_query_path = _queries_dir / "get_input_edges.sparql" -_get_output_edges_query_path = _queries_dir / "get_output_edges.sparql" -_get_root_query_path = _queries_dir / "get_root.sparql" + +def _get_inner_edges_query() -> str: + return files("cwltool").joinpath("rdfqueries/get_inner_edges.sparql").read_text() + + +def _get_input_edges_query() -> str: + return files("cwltool").joinpath("rdfqueries/get_input_edges.sparql").read_text() + + +def _get_output_edges_query() -> str: + return files("cwltool").joinpath("rdfqueries/get_output_edges.sparql").read_text() + + +def _get_root_query() -> str: + return files("cwltool").joinpath("rdfqueries/get_root.sparql").read_text() class CWLViewer: @@ -32,8 +43,7 @@ def _load_cwl_graph(self, rdf_description: str) -> rdflib.graph.Graph: return rdf_graph def _set_inner_edges(self) -> None: - with open(_get_inner_edges_query_path) as f: - get_inner_edges_query = f.read() + get_inner_edges_query = _get_inner_edges_query() inner_edges = cast( Iterator[rdflib.query.ResultRow], self._rdf_graph.query( @@ -95,8 +105,7 @@ def _set_inner_edges(self) -> None: ) def _set_input_edges(self) -> None: - with open(_get_input_edges_query_path) as f: - get_input_edges_query = f.read() + get_input_edges_query = _get_input_edges_query() inputs_subgraph = pydot.Subgraph(graph_name="cluster_inputs") self._dot_graph.add_subgraph(inputs_subgraph) inputs_subgraph.set("rank", "same") @@ -123,8 +132,7 @@ def _set_input_edges(self) -> None: self._dot_graph.add_edge(pydot.Edge(str(input_row["input"]), str(input_row["step"]))) def _set_output_edges(self) -> None: - with open(_get_output_edges_query_path) as f: - get_output_edges = f.read() + get_output_edges = _get_output_edges_query() outputs_graph = pydot.Subgraph(graph_name="cluster_outputs") self._dot_graph.add_subgraph(outputs_graph) outputs_graph.set("rank", "same") @@ -151,10 +159,9 @@ def _set_output_edges(self) -> None: self._dot_graph.add_edge(pydot.Edge(output_edge_row["step"], output_edge_row["output"])) def _get_root_graph_uri(self) -> rdflib.term.Identifier: - with open(_get_root_query_path) as f: - get_root_query = f.read() + get_root_query = _get_root_query() root = cast( - List[rdflib.query.ResultRow], + list[rdflib.query.ResultRow], list( self._rdf_graph.query( get_root_query, diff --git a/cwltool/docker.py b/cwltool/docker.py index d0f628b15..b03ae635c 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -9,8 +9,9 @@ import subprocess # nosec import sys import threading +from collections.abc import MutableMapping from io import StringIO # pylint: disable=redefined-builtin -from typing import Callable, Dict, List, MutableMapping, Optional, Set, Tuple, cast +from typing import Callable, Optional, cast import requests @@ -23,13 +24,13 @@ from .pathmapper import MapperEnt, PathMapper from .utils import CWLObjectType, create_tmp_dir, ensure_writable -_IMAGES: Set[str] = set() +_IMAGES: set[str] = set() _IMAGES_LOCK = threading.Lock() -__docker_machine_mounts: Optional[List[str]] = None +__docker_machine_mounts: Optional[list[str]] = None __docker_machine_mounts_lock = threading.Lock() -def _get_docker_machine_mounts() -> List[str]: +def _get_docker_machine_mounts() -> list[str]: global __docker_machine_mounts if __docker_machine_mounts is None: with __docker_machine_mounts_lock: @@ -83,9 +84,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize a command line builder using the Docker software container engine.""" @@ -94,7 +95,7 @@ def __init__( def get_image( self, - docker_requirement: Dict[str, str], + docker_requirement: dict[str, str], pull_image: bool, force_pull: bool, tmp_outdir_prefix: str, @@ -127,7 +128,7 @@ def get_image( except (OSError, subprocess.CalledProcessError, UnicodeError): pass - cmd: List[str] = [] + cmd: list[str] = [] if "dockerFile" in docker_requirement: dockerfile_dir = create_tmp_dir(tmp_outdir_prefix) with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile: @@ -204,13 +205,13 @@ def get_from_requirements( if not shutil.which(self.docker_exec): raise WorkflowException(f"{self.docker_exec} executable is not available") - if self.get_image(cast(Dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix): + if self.get_image(cast(dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix): return cast(Optional[str], r["dockerImageId"]) raise WorkflowException("Docker image %s not found" % r["dockerImageId"]) @staticmethod def append_volume( - runtime: List[str], + runtime: list[str], source: str, target: str, writable: bool = False, @@ -233,7 +234,7 @@ def append_volume( os.makedirs(source) def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] ) -> None: """Append volume a file/dir mapping to the runtime option list.""" if not volume.resolved.startswith("_:"): @@ -242,7 +243,7 @@ def add_file_or_directory_volume( def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -266,7 +267,7 @@ def add_writable_file_volume( def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -295,7 +296,7 @@ def add_writable_directory_volume( shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt or new_dir) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: # spec currently says "HOME must be set to the designated output # directory." but spec might change to designated temp directory. # runtime.append("--env=HOME=/tmp") @@ -306,7 +307,7 @@ def _required_env(self) -> Dict[str, str]: def create_runtime( self, env: MutableMapping[str, str], runtimeContext: RuntimeContext - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], Optional[str]]: any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False user_space_docker_cmd = runtimeContext.user_space_docker_cmd if user_space_docker_cmd: @@ -445,9 +446,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize a command line builder using the Podman software container engine.""" diff --git a/cwltool/docker_id.py b/cwltool/docker_id.py index bb436b2cb..90484b686 100644 --- a/cwltool/docker_id.py +++ b/cwltool/docker_id.py @@ -1,10 +1,10 @@ """Helper functions for docker.""" import subprocess # nosec -from typing import List, Optional, Tuple +from typing import Optional -def docker_vm_id() -> Tuple[Optional[int], Optional[int]]: +def docker_vm_id() -> tuple[Optional[int], Optional[int]]: """ Return the User ID and Group ID of the default docker user inside the VM. @@ -21,7 +21,7 @@ def docker_vm_id() -> Tuple[Optional[int], Optional[int]]: return (None, None) -def check_output_and_strip(cmd: List[str]) -> Optional[str]: +def check_output_and_strip(cmd: list[str]) -> Optional[str]: """ Pass a command list to :py:func:`subprocess.check_output`. @@ -48,7 +48,7 @@ def docker_machine_name() -> Optional[str]: return check_output_and_strip(["docker-machine", "active"]) -def cmd_output_matches(check_cmd: List[str], expected_status: str) -> bool: +def cmd_output_matches(check_cmd: list[str], expected_status: str) -> bool: """ Run a command and compares output to expected. @@ -80,7 +80,7 @@ def docker_machine_running() -> bool: return cmd_output_matches(["docker-machine", "status", machine_name], "Running") -def cmd_output_to_int(cmd: List[str]) -> Optional[int]: +def cmd_output_to_int(cmd: list[str]) -> Optional[int]: """ Run the provided command and returns the integer value of the result. @@ -97,7 +97,7 @@ def cmd_output_to_int(cmd: List[str]) -> Optional[int]: return None -def boot2docker_id() -> Tuple[Optional[int], Optional[int]]: +def boot2docker_id() -> tuple[Optional[int], Optional[int]]: """ Get the UID and GID of the docker user inside a running boot2docker vm. @@ -108,7 +108,7 @@ def boot2docker_id() -> Tuple[Optional[int], Optional[int]]: return (uid, gid) -def docker_machine_id() -> Tuple[Optional[int], Optional[int]]: +def docker_machine_id() -> tuple[Optional[int], Optional[int]]: """ Ask docker-machine for active machine and gets the UID of the docker user. diff --git a/cwltool/env_to_stdout.py b/cwltool/env_to_stdout.py index 33b832479..0309fe08f 100644 --- a/cwltool/env_to_stdout.py +++ b/cwltool/env_to_stdout.py @@ -11,10 +11,9 @@ """ import os -from typing import Dict -def deserialize_env(data: str) -> Dict[str, str]: +def deserialize_env(data: str) -> dict[str, str]: """Deserialize the output of `env -0` to dictionary.""" result = {} for item in data.strip("\0").split("\0"): diff --git a/cwltool/errors.py b/cwltool/errors.py index 045b9b383..2b7e50aed 100644 --- a/cwltool/errors.py +++ b/cwltool/errors.py @@ -11,6 +11,13 @@ from cwl_utils.errors import GraphTargetMissingException as GraphTargetMissingException from cwl_utils.errors import WorkflowException as WorkflowException +__all__ = ( + "GraphTargetMissingException", + "WorkflowException", + "UnsupportedRequirement", + "ArgumentException", +) + class UnsupportedRequirement(WorkflowException): pass diff --git a/cwltool/executors.py b/cwltool/executors.py index bfc87f9c7..9d0559726 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -7,18 +7,9 @@ import os import threading from abc import ABCMeta, abstractmethod +from collections.abc import Iterable, MutableSequence from threading import Lock -from typing import ( - Dict, - Iterable, - List, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from typing import Optional, Union, cast import psutil from mypy_extensions import mypyc_attr @@ -28,7 +19,6 @@ from .command_line_tool import CallbackJob, ExpressionJob from .context import RuntimeContext, getdefault from .cuda import cuda_version_and_device_count -from .cwlprov.provenance_profile import ProvenanceProfile from .errors import WorkflowException from .job import JobBase from .loghandler import _logger @@ -50,8 +40,8 @@ class JobExecutor(metaclass=ABCMeta): def __init__(self) -> None: """Initialize.""" self.final_output: MutableSequence[Optional[CWLObjectType]] = [] - self.final_status: List[str] = [] - self.output_dirs: Set[str] = set() + self.final_status: list[str] = [] + self.output_dirs: set[str] = set() def __call__( self, @@ -59,7 +49,7 @@ def __call__( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: logging.Logger = _logger, - ) -> Tuple[Optional[CWLObjectType], str]: + ) -> tuple[Optional[CWLObjectType], str]: return self.execute(process, job_order_object, runtime_context, logger) def output_callback(self, out: Optional[CWLObjectType], process_status: str) -> None: @@ -83,7 +73,7 @@ def execute( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: logging.Logger = _logger, - ) -> Tuple[Union[Optional[CWLObjectType]], str]: + ) -> tuple[Union[Optional[CWLObjectType]], str]: """Execute the process.""" self.final_output = [] @@ -112,7 +102,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: runtime_context.toplevel = True runtime_context.workflow_eval_lock = threading.Condition(threading.RLock()) - job_reqs: Optional[List[CWLObjectType]] = None + job_reqs: Optional[list[CWLObjectType]] = None if "https://w3id.org/cwl/cwl#requirements" in job_order_object: if process.metadata.get(ORIGINAL_CWLVERSION) == "v1.0": raise WorkflowException( @@ -121,7 +111,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: "can set the cwlVersion to v1.1" ) job_reqs = cast( - List[CWLObjectType], + list[CWLObjectType], job_order_object["https://w3id.org/cwl/cwl#requirements"], ) elif "cwl:defaults" in process.metadata and "https://w3id.org/cwl/cwl#requirements" in cast( @@ -134,7 +124,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: "can set the cwlVersion to v1.1" ) job_reqs = cast( - Optional[List[CWLObjectType]], + Optional[list[CWLObjectType]], cast(CWLObjectType, process.metadata["cwl:defaults"])[ "https://w3id.org/cwl/cwl#requirements" ], @@ -203,11 +193,13 @@ def run_jobs( # define provenance profile for single commandline tool if not isinstance(process, Workflow) and runtime_context.research_obj is not None: - process.provenance_object = ProvenanceProfile( - runtime_context.research_obj, + process.provenance_object = runtime_context.research_obj.initialize_provenance( full_name=runtime_context.cwl_full_name, - host_provenance=False, - user_provenance=False, + # following are only set from main when directly command line tool + # when nested in a workflow, they should be disabled since they would + # already have been provided/initialized by the parent workflow prov-obj + host_provenance=runtime_context.prov_host, + user_provenance=runtime_context.prov_user, orcid=runtime_context.orcid, # single tool execution, so RO UUID = wf UUID = tool UUID run_uuid=runtime_context.research_obj.ro_uuid, @@ -277,22 +269,22 @@ class MultithreadedJobExecutor(JobExecutor): def __init__(self) -> None: """Initialize.""" super().__init__() - self.exceptions: List[WorkflowException] = [] - self.pending_jobs: List[JobsType] = [] + self.exceptions: list[WorkflowException] = [] + self.pending_jobs: list[JobsType] = [] self.pending_jobs_lock = threading.Lock() self.max_ram = int(psutil.virtual_memory().available / 2**20) - self.max_cores = float(psutil.cpu_count()) + self.max_cores = float(psutil.cpu_count() or 1) self.max_cuda = cuda_version_and_device_count()[1] self.allocated_ram = float(0) self.allocated_cores = float(0) self.allocated_cuda: int = 0 def select_resources( - self, request: Dict[str, Union[int, float]], runtime_context: RuntimeContext - ) -> Dict[str, Union[int, float]]: # pylint: disable=unused-argument + self, request: dict[str, Union[int, float]], runtime_context: RuntimeContext + ) -> dict[str, Union[int, float]]: # pylint: disable=unused-argument """NaĂ¯ve check for available cpu cores and memory.""" - result: Dict[str, Union[int, float]] = {} + result: dict[str, Union[int, float]] = {} maxrsc = {"cores": self.max_cores, "ram": self.max_ram} resources_types = {"cores", "ram"} if "cudaDeviceCountMin" in request or "cudaDeviceCountMax" in request: @@ -334,7 +326,10 @@ def _runner( self.exceptions.append(err) except Exception as err: # pylint: disable=broad-except _logger.exception(f"Got workflow error: {err}") - self.exceptions.append(WorkflowException(str(err))) + wf_exc = WorkflowException(str(err)) + wf_exc.__cause__ = err + wf_exc.__suppress_context__ = True + self.exceptions.append(wf_exc) finally: if runtime_context.workflow_eval_lock: with runtime_context.workflow_eval_lock: @@ -438,7 +433,7 @@ def run_jobs( logger: logging.Logger, runtime_context: RuntimeContext, ) -> None: - self.taskqueue: TaskQueue = TaskQueue(threading.Lock(), psutil.cpu_count()) + self.taskqueue: TaskQueue = TaskQueue(threading.Lock(), int(math.ceil(self.max_cores))) try: jobiter = process.job(job_order_object, self.output_callback, runtime_context) @@ -491,5 +486,5 @@ def execute( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: Optional[logging.Logger] = None, - ) -> Tuple[Optional[CWLObjectType], str]: + ) -> tuple[Optional[CWLObjectType], str]: return {}, "success" diff --git a/cwltool/extensions-v1.2.yml b/cwltool/extensions-v1.2.yml index c39b15d07..ae371c671 100644 --- a/cwltool/extensions-v1.2.yml +++ b/cwltool/extensions-v1.2.yml @@ -236,7 +236,7 @@ $graph: name: LoopOutputModes symbols: [ last, all ] default: last - doc: + doc: | - Specify the desired method of dealing with loop outputs - Default. Propagates only the last computed element to the subsequent steps when the loop terminates. - Propagates a single array with all output values to the subsequent steps when the loop terminates. diff --git a/cwltool/factory.py b/cwltool/factory.py index 85d7344e6..eaf98e3cf 100644 --- a/cwltool/factory.py +++ b/cwltool/factory.py @@ -1,5 +1,5 @@ import os -from typing import Any, Dict, Optional, Union +from typing import Any, Optional, Union from . import load_tool from .context import LoadingContext, RuntimeContext @@ -62,7 +62,7 @@ def __init__( else: self.loading_context = loading_context - def make(self, cwl: Union[str, Dict[str, Any]]) -> Callable: + def make(self, cwl: Union[str, dict[str, Any]]) -> Callable: """Instantiate a CWL object from a CWl document.""" load = load_tool.load_tool(cwl, self.loading_context) if isinstance(load, int): diff --git a/cwltool/flatten.py b/cwltool/flatten.py index 420d90d04..3c057ebbe 100644 --- a/cwltool/flatten.py +++ b/cwltool/flatten.py @@ -1,12 +1,17 @@ -from typing import Any, Callable, List, cast +""" +Our version of the popular flatten() method. -# http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html +http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html +""" +from typing import Any, Callable, cast -def flatten(thing, ltypes=(list, tuple)): - # type: (Any, Any) -> List[Any] + +def flatten(thing: Any) -> list[Any]: + """Flatten a list without recursion problems.""" if thing is None: return [] + ltypes = (list, tuple) if not isinstance(thing, ltypes): return [thing] @@ -22,4 +27,4 @@ def flatten(thing, ltypes=(list, tuple)): else: thing_list[i : i + 1] = thing_list[i] i += 1 - return cast(Callable[[Any], List[Any]], ltype)(thing_list) + return cast(Callable[[Any], list[Any]], ltype)(thing_list) diff --git a/cwltool/job.py b/cwltool/job.py index 817cb04c0..2c6bb9f77 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -5,6 +5,7 @@ import math import os import re +import shlex import shutil import signal import stat @@ -15,27 +16,12 @@ import time import uuid from abc import ABCMeta, abstractmethod +from collections.abc import Iterable, Mapping, MutableMapping, MutableSequence +from re import Match from threading import Timer -from typing import ( - IO, - TYPE_CHECKING, - Callable, - Dict, - Iterable, - List, - Mapping, - Match, - MutableMapping, - MutableSequence, - Optional, - TextIO, - Tuple, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Callable, Optional, TextIO, Union, cast import psutil -import shellescape from prov.model import PROV from schema_salad.sourceline import SourceLine from schema_salad.utils import json_dump, json_dumps @@ -122,9 +108,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize the job object.""" @@ -140,7 +126,7 @@ def __init__( self.requirements = requirements self.hints = hints self.name = name - self.command_line: List[str] = [] + self.command_line: list[str] = [] self.pathmapper = PathMapper([], "", "") self.make_path_mapper = make_path_mapper self.generatemapper: Optional[PathMapper] = None @@ -228,7 +214,7 @@ def is_streamable(file: str) -> bool: def _execute( self, - runtime: List[str], + runtime: list[str], env: MutableMapping[str, str], runtimeContext: RuntimeContext, monitor_function: Optional[Callable[["subprocess.Popen[str]"], None]] = None, @@ -271,7 +257,7 @@ def _execute( self.outdir, " \\\n ".join( [ - shellescape.quote(str(arg)) if shouldquote(str(arg)) else str(arg) + shlex.quote(str(arg)) if shouldquote(str(arg)) else str(arg) for arg in (runtime + self.command_line) ] ), @@ -321,7 +307,7 @@ def stderr_stdout_log_path( commands = [str(x) for x in runtime + self.command_line] if runtimeContext.secret_store is not None: commands = cast( - List[str], + list[str], runtimeContext.secret_store.retrieve(cast(CWLOutputType, commands)), ) env = cast( @@ -390,17 +376,30 @@ def stderr_stdout_log_path( except OSError as e: if e.errno == 2: if runtime: - _logger.error("'%s' not found: %s", runtime[0], str(e)) + _logger.error( + "'%s' not found: %s", runtime[0], str(e), exc_info=runtimeContext.debug + ) else: - _logger.error("'%s' not found: %s", self.command_line[0], str(e)) + _logger.error( + "'%s' not found: %s", + self.command_line[0], + str(e), + exc_info=runtimeContext.debug, + ) else: - _logger.exception("Exception while running job") + _logger.exception( + "Exception while running job: %s", str(e), exc_info=runtimeContext.debug + ) processStatus = "permanentFail" except WorkflowException as err: - _logger.error("[job %s] Job error:\n%s", self.name, str(err)) + _logger.error( + "[job %s] Job error:\n%s", self.name, str(err), exc_info=runtimeContext.debug + ) processStatus = "permanentFail" - except Exception: - _logger.exception("Exception while running job") + except Exception as err: + _logger.exception( + "Exception while running job: %s.", str(err), exc_info=runtimeContext.debug + ) processStatus = "permanentFail" if ( runtimeContext.research_obj is not None @@ -456,7 +455,7 @@ def stderr_stdout_log_path( shutil.rmtree(self.tmpdir, True) @abstractmethod - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: """Variables required by the CWL spec (HOME, TMPDIR, etc). Note that with containers, the paths will (likely) be those from @@ -481,7 +480,7 @@ def prepare_environment( applied (in that order). """ # Start empty - env: Dict[str, str] = {} + env: dict[str, str] = {} # Preserve any env vars if runtimeContext.preserve_entire_environment: @@ -589,7 +588,7 @@ def run( self._execute([], self.environment, runtimeContext, monitor_function) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: env = {} env["HOME"] = self.outdir env["TMPDIR"] = self.tmpdir @@ -623,24 +622,24 @@ def create_runtime( self, env: MutableMapping[str, str], runtime_context: RuntimeContext, - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], Optional[str]]: """Return the list of commands to run the selected container engine.""" @staticmethod @abstractmethod - def append_volume(runtime: List[str], source: str, target: str, writable: bool = False) -> None: + def append_volume(runtime: list[str], source: str, target: str, writable: bool = False) -> None: """Add binding arguments to the runtime list.""" @abstractmethod def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] ) -> None: """Append volume a file/dir mapping to the runtime option list.""" @abstractmethod def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -650,7 +649,7 @@ def add_writable_file_volume( @abstractmethod def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -674,7 +673,7 @@ def _preserve_environment_on_containers_warning( def create_file_and_add_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], secret_store: Optional[SecretStore], @@ -706,7 +705,7 @@ def create_file_and_add_volume( def add_volumes( self, pathmapper: PathMapper, - runtime: List[str], + runtime: list[str], tmpdir_prefix: str, secret_store: Optional[SecretStore] = None, any_path_okay: bool = False, @@ -809,7 +808,7 @@ def run( ) except Exception as err: container = "Singularity" if runtimeContext.singularity else "Docker" - _logger.debug("%s error", container, exc_info=True) + _logger.debug("%s error", container, exc_info=runtimeContext.debug) if docker_is_req: raise UnsupportedRequirement( f"{container} is required to run this tool: {str(err)}" @@ -918,7 +917,7 @@ def docker_monitor( def _job_popen( - commands: List[str], + commands: list[str], stdin_path: Optional[str], stdout_path: Optional[str], stderr_path: Optional[str], diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index d6352f918..f7f4936cd 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -7,18 +7,9 @@ import re import urllib import uuid +from collections.abc import MutableMapping, MutableSequence from functools import partial -from typing import ( - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from typing import Any, Optional, Union, cast from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -93,7 +84,7 @@ def resolve_tool_uri( resolver: Optional[ResolverType] = None, fetcher_constructor: Optional[FetcherCallableType] = None, document_loader: Optional[Loader] = None, -) -> Tuple[str, str]: +) -> tuple[str, str]: uri = None # type: Optional[str] split = urllib.parse.urlsplit(argsworkflow) # In case of Windows path, urlsplit misjudge Drive letters as scheme, here we are skipping that @@ -117,7 +108,7 @@ def resolve_tool_uri( def fetch_document( argsworkflow: Union[str, CWLObjectType], loadingContext: Optional[LoadingContext] = None, -) -> Tuple[LoadingContext, CommentedMap, str]: +) -> tuple[LoadingContext, CommentedMap, str]: """Retrieve a CWL document.""" if loadingContext is None: loadingContext = LoadingContext() @@ -144,14 +135,14 @@ def fetch_document( return loadingContext, workflowobj, uri if isinstance(argsworkflow, MutableMapping): uri = cast(str, argsworkflow["id"]) if argsworkflow.get("id") else "_:" + str(uuid.uuid4()) - workflowobj = cast(CommentedMap, cmap(cast(Dict[str, Any], argsworkflow), fn=uri)) + workflowobj = cast(CommentedMap, cmap(cast(dict[str, Any], argsworkflow), fn=uri)) loadingContext.loader.idx[uri] = workflowobj return loadingContext, workflowobj, uri raise ValidationException("Must be URI or object: '%s'" % argsworkflow) def _convert_stdstreams_to_files( - workflowobj: Union[CWLObjectType, MutableSequence[Union[CWLObjectType, str, int]], str] + workflowobj: Union[CWLObjectType, MutableSequence[Union[CWLObjectType, str, int]], str], ) -> None: if isinstance(workflowobj, MutableMapping): if workflowobj.get("class") == "CommandLineTool": @@ -228,7 +219,7 @@ def _convert_stdstreams_to_files( def _add_blank_ids( - workflowobj: Union[CWLObjectType, MutableSequence[Union[CWLObjectType, str]]] + workflowobj: Union[CWLObjectType, MutableSequence[Union[CWLObjectType, str]]], ) -> None: if isinstance(workflowobj, MutableMapping): if ( @@ -256,7 +247,7 @@ def _add_blank_ids( def _fast_parser_convert_stdstreams_to_files( - processobj: Union[cwl_v1_2.Process, MutableSequence[cwl_v1_2.Process]] + processobj: Union[cwl_v1_2.Process, MutableSequence[cwl_v1_2.Process]], ) -> None: if isinstance(processobj, cwl_v1_2.CommandLineTool): cwl_v1_2_utils.convert_stdstreams_to_files(processobj) @@ -306,7 +297,7 @@ def fast_parser( uri: str, loadingContext: LoadingContext, fetcher: Fetcher, -) -> Tuple[Union[CommentedMap, CommentedSeq], CommentedMap]: +) -> tuple[Union[CommentedMap, CommentedSeq], CommentedMap]: lopt = cwl_v1_2.LoadingOptions(idx=loadingContext.codegen_idx, fileuri=fileuri, fetcher=fetcher) if uri not in loadingContext.codegen_idx: @@ -326,7 +317,7 @@ def fast_parser( processobj = cwl_v1_2.save(objects, relative_uris=False) - metadata: Dict[str, Any] = {} + metadata: dict[str, Any] = {} metadata["id"] = loadopt.fileuri if loadopt.namespaces: @@ -353,7 +344,7 @@ def fast_parser( objects, loadopt = loadingContext.codegen_idx[nofrag] fileobj = cmap( cast( - Union[int, float, str, Dict[str, Any], List[Any], None], + Union[int, float, str, dict[str, Any], list[Any], None], cwl_v1_2.save(objects, relative_uris=False), ) ) @@ -370,7 +361,7 @@ def fast_parser( return cast( Union[CommentedMap, CommentedSeq], - cmap(cast(Union[Dict[str, Any], List[Any]], processobj)), + cmap(cast(Union[dict[str, Any], list[Any]], processobj)), ), cast(CommentedMap, cmap(metadata)) @@ -379,7 +370,7 @@ def resolve_and_validate_document( workflowobj: Union[CommentedMap, CommentedSeq], uri: str, preprocess_only: bool = False, -) -> Tuple[LoadingContext, str]: +) -> tuple[LoadingContext, str]: """Validate a CWL document.""" if not loadingContext.loader: raise ValueError("loadingContext must have a loader.") @@ -394,7 +385,7 @@ def resolve_and_validate_document( if "cwl:tool" in workflowobj: jobobj, _ = loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) - del cast(Dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"] + del cast(dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, loadingContext)[1] @@ -624,17 +615,18 @@ def resolve_overrides( ov: IdxResultType, ov_uri: str, baseurl: str, -) -> List[CWLObjectType]: +) -> list[CWLObjectType]: ovloader = Loader(overrides_ctx) ret, _ = ovloader.resolve_all(ov, baseurl) if not isinstance(ret, CommentedMap): raise Exception("Expected CommentedMap, got %s" % type(ret)) cwl_docloader = get_schema("v1.0")[0] cwl_docloader.resolve_all(ret, ov_uri) - return cast(List[CWLObjectType], ret["http://commonwl.org/cwltool#overrides"]) + return cast(list[CWLObjectType], ret["http://commonwl.org/cwltool#overrides"]) -def load_overrides(ov: str, base_url: str) -> List[CWLObjectType]: +def load_overrides(ov: str, base_url: str) -> list[CWLObjectType]: + """Load and resolve any overrides.""" ovloader = Loader(overrides_ctx) return resolve_overrides(ovloader.fetch(ov), ov, base_url) @@ -644,7 +636,7 @@ def recursive_resolve_and_validate_document( workflowobj: Union[CommentedMap, CommentedSeq], uri: str, preprocess_only: bool = False, -) -> Tuple[LoadingContext, str, Process]: +) -> tuple[LoadingContext, str, Process]: """Validate a CWL document, checking that a tool object can be built.""" loadingContext, uri = resolve_and_validate_document( loadingContext, diff --git a/cwltool/loghandler.py b/cwltool/loghandler.py index 76daa8be9..c76830816 100644 --- a/cwltool/loghandler.py +++ b/cwltool/loghandler.py @@ -11,7 +11,7 @@ def configure_logging( - stderr_handler: logging.Handler, + err_handler: logging.Handler, no_warnings: bool, quiet: bool, debug: bool, @@ -21,25 +21,29 @@ def configure_logging( ) -> None: """Configure logging.""" rdflib_logger = logging.getLogger("rdflib.term") - rdflib_logger.addHandler(stderr_handler) + rdflib_logger.addHandler(err_handler) rdflib_logger.setLevel(logging.ERROR) deps_logger = logging.getLogger("galaxy.tool_util.deps") - deps_logger.addHandler(stderr_handler) + deps_logger.addHandler(err_handler) ss_logger = logging.getLogger("salad") - ss_logger.addHandler(stderr_handler) if no_warnings: - stderr_handler.setLevel(logging.ERROR) - if quiet: + err_handler.setLevel(logging.ERROR) + ss_logger.setLevel(logging.ERROR) + elif quiet: # Silence STDERR, not an eventual provenance log file - stderr_handler.setLevel(logging.WARN) + err_handler.setLevel(logging.WARN) + ss_logger.setLevel(logging.WARN) + else: + err_handler.setLevel(logging.INFO) + ss_logger.setLevel(logging.INFO) if debug: # Increase to debug for both stderr and provenance log file base_logger.setLevel(logging.DEBUG) - stderr_handler.setLevel(logging.DEBUG) + err_handler.setLevel(logging.DEBUG) rdflib_logger.setLevel(logging.DEBUG) deps_logger.setLevel(logging.DEBUG) fmtclass = coloredlogs.ColoredFormatter if enable_color else logging.Formatter formatter = fmtclass("%(levelname)s %(message)s") if timestamps: formatter = fmtclass("[%(asctime)s] %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S") - stderr_handler.setFormatter(formatter) + err_handler.setFormatter(formatter) diff --git a/cwltool/main.py b/cwltool/main.py index 30f299f09..90cb2e2c8 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -15,26 +15,15 @@ import urllib import warnings from codecs import getwriter -from typing import ( - IO, - Any, - Callable, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Sized, - Tuple, - Union, - cast, -) +from collections.abc import Mapping, MutableMapping, MutableSequence, Sized +from importlib.resources import files +from typing import IO, Any, Callable, Optional, Union, cast import argcomplete import coloredlogs import requests import ruamel.yaml +from rich_argparse import RichHelpFormatter from ruamel.yaml.comments import CommentedMap, CommentedSeq from ruamel.yaml.main import YAML from schema_salad.exceptions import ValidationException @@ -108,7 +97,6 @@ CWLOutputType, HasReqsHints, adjustDirObjs, - files, normalizeFilesDirs, processes_to_kill, trim_listing, @@ -185,11 +173,11 @@ def append_word_to_default_user_agent(word: str) -> None: def generate_example_input( inptype: Optional[CWLOutputType], default: Optional[CWLOutputType], -) -> Tuple[Any, str]: +) -> tuple[Any, str]: """Convert a single input schema into an example.""" example = None comment = "" - defaults = { + defaults: CWLObjectType = { "null": "null", "Any": "null", "boolean": False, @@ -202,7 +190,7 @@ def generate_example_input( "Directory": ruamel.yaml.comments.CommentedMap( [("class", "Directory"), ("path", "a/directory/path")] ), - } # type: CWLObjectType + } if isinstance(inptype, MutableSequence): optional = False if "null" in inptype: @@ -244,7 +232,7 @@ def generate_example_input( if default is not None: example = default elif inptype["type"] == "enum": - symbols = cast(List[str], inptype["symbols"]) + symbols = cast(list[str], inptype["symbols"]) if default is not None: example = default elif "default" in inptype: @@ -260,7 +248,7 @@ def generate_example_input( comment = '"{}" record type.'.format(inptype["name"]) else: comment = "Anonymous record type." - for field in cast(List[CWLObjectType], inptype["fields"]): + for field in cast(list[CWLObjectType], inptype["fields"]): value, f_comment = generate_example_input(field["type"], None) example.insert(0, shortname(cast(str, field["name"])), value, f_comment) elif "default" in inptype: @@ -343,10 +331,10 @@ def generate_input_template(tool: Process) -> CWLObjectType: """Generate an example input object for the given CWL process.""" template = ruamel.yaml.comments.CommentedMap() for inp in cast( - List[MutableMapping[str, str]], + list[CWLObjectType], realize_input_schema(tool.tool["inputs"], tool.schemaDefs), ): - name = shortname(inp["id"]) + name = shortname(cast(str, inp["id"])) value, comment = generate_example_input(inp["type"], inp.get("default", None)) template.insert(0, name, value, comment) return template @@ -356,9 +344,9 @@ def load_job_order( args: argparse.Namespace, stdin: IO[Any], fetcher_constructor: Optional[FetcherCallableType], - overrides_list: List[CWLObjectType], + overrides_list: list[CWLObjectType], tool_file_uri: str, -) -> Tuple[Optional[CWLObjectType], str, Loader]: +) -> tuple[Optional[CWLObjectType], str, Loader]: job_order_object = None job_order_file = None @@ -423,10 +411,13 @@ def init_job_order( ) -> CWLObjectType: secrets_req, _ = process.get_requirement("http://commonwl.org/cwltool#Secrets") if job_order_object is None: - namemap = {} # type: Dict[str, str] - records = [] # type: List[str] + namemap: dict[str, str] = {} + records: list[str] = [] toolparser = generate_parser( - argparse.ArgumentParser(prog=args.workflow), + argparse.ArgumentParser( + prog=args.workflow, + formatter_class=RichHelpFormatter, + ), process, namemap, records, @@ -463,7 +454,7 @@ def init_job_order( if secret_store and secrets_req: secret_store.store( - [shortname(sc) for sc in cast(List[str], secrets_req["secrets"])], + [shortname(sc) for sc in cast(list[str], secrets_req["secrets"])], job_order_object, ) @@ -486,7 +477,7 @@ def path_to_loc(p: CWLObjectType) -> None: p["location"] = p["path"] del p["path"] - ns = {} # type: ContextType + ns: ContextType = {} ns.update(cast(ContextType, job_order_object.get("$namespaces", {}))) ns.update(cast(ContextType, process.metadata.get("$namespaces", {}))) ld = Loader(ns) @@ -532,7 +523,7 @@ def expand_formats(p: CWLObjectType) -> None: if secret_store and secrets_req: secret_store.store( - [shortname(sc) for sc in cast(List[str], secrets_req["secrets"])], + [shortname(sc) for sc in cast(list[str], secrets_req["secrets"])], job_order_object, ) @@ -583,7 +574,7 @@ def prov_deps( def remove_non_cwl(deps: CWLObjectType) -> None: if "secondaryFiles" in deps: - sec_files = cast(List[CWLObjectType], deps["secondaryFiles"]) + sec_files = cast(list[CWLObjectType], deps["secondaryFiles"]) for index, entry in enumerate(sec_files): if not ("format" in entry and entry["format"] == CWL_IANA): del sec_files[index] @@ -602,11 +593,11 @@ def find_deps( nestdirs: bool = True, ) -> CWLObjectType: """Find the dependencies of the CWL document.""" - deps = { + deps: CWLObjectType = { "class": "File", "location": uri, "format": CWL_IANA, - } # type: CWLObjectType + } def loadref(base: str, uri: str) -> Union[CommentedMap, CommentedSeq, str, None]: return document_loader.fetch(document_loader.fetcher.urljoin(base, uri)) @@ -638,7 +629,8 @@ def print_pack( return json_dumps(target, indent=4, default=str) -def supported_cwl_versions(enable_dev: bool) -> List[str]: +def supported_cwl_versions(enable_dev: bool) -> list[str]: + """Return a list of currently supported CWL versions.""" # ALLUPDATES and UPDATES are dicts if enable_dev: versions = list(ALLUPDATES) @@ -692,8 +684,8 @@ def formatTime(self, record: logging.LogRecord, datefmt: Optional[str] = None) - def setup_provenance( args: argparse.Namespace, runtimeContext: RuntimeContext, - argsl: Optional[List[str]] = None, -) -> Tuple[ProvOut, "logging.StreamHandler[ProvOut]"]: + argsl: Optional[list[str]] = None, +) -> tuple[ProvOut, "logging.StreamHandler[ProvOut]"]: if not args.compute_checksum: _logger.error("--provenance incompatible with --no-compute-checksum") raise ArgumentException() @@ -940,7 +932,7 @@ def print_targets( _logger.info("%s steps targets:", prefix[:-1]) for t in tool.tool["steps"]: print(f" {prefix}{shortname(t['id'])}", file=stdout) - run: Union[str, Process, Dict[str, Any]] = t["run"] + run: Union[str, Process, dict[str, Any]] = t["run"] if isinstance(run, str): process = make_tool(run, loading_context) elif isinstance(run, dict): @@ -951,7 +943,7 @@ def print_targets( def main( - argsl: Optional[List[str]] = None, + argsl: Optional[list[str]] = None, args: Optional[argparse.Namespace] = None, job_order_object: Optional[CWLObjectType] = None, stdin: IO[Any] = sys.stdin, @@ -979,12 +971,6 @@ def main( stdout = cast(IO[str], stdout) _logger.removeHandler(defaultStreamHandler) - stderr_handler = logger_handler - if stderr_handler is not None: - _logger.addHandler(stderr_handler) - else: - coloredlogs.install(logger=_logger, stream=stderr) - stderr_handler = _logger.handlers[-1] workflowobj = None prov_log_handler: Optional[logging.StreamHandler[ProvOut]] = None global docker_exe @@ -994,11 +980,12 @@ def main( user_agent += f" {progname}" # append the real program name as well append_word_to_default_user_agent(user_agent) + err_handler: logging.Handler = defaultStreamHandler try: if args is None: if argsl is None: argsl = sys.argv[1:] - addl = [] # type: List[str] + addl: list[str] = [] if "CWLTOOL_OPTIONS" in os.environ: c_opts = os.environ["CWLTOOL_OPTIONS"].split(" ") addl = [x for x in c_opts if x != ""] @@ -1009,6 +996,13 @@ def main( if not args.cidfile_dir: args.cidfile_dir = os.getcwd() del args.record_container_id + if logger_handler is not None: + err_handler = logger_handler + _logger.addHandler(err_handler) + else: + coloredlogs.install(logger=_logger, stream=stdout if args.validate else stderr) + err_handler = _logger.handlers[-1] + logging.getLogger("salad").handlers = _logger.handlers if runtimeContext is None: runtimeContext = RuntimeContext(vars(args)) @@ -1027,7 +1021,7 @@ def main( setattr(args, key, val) configure_logging( - stderr_handler, + err_handler, args.no_warnings, args.quiet, runtimeContext.debug, @@ -1071,6 +1065,11 @@ def main( loadingContext = setup_loadingContext(loadingContext, runtimeContext, args) + if loadingContext.research_obj: + # early forward parameters required for a single command line tool + runtimeContext.prov_host = loadingContext.host_provenance + runtimeContext.prov_user = loadingContext.user_provenance + uri, tool_file_uri = resolve_tool_uri( args.workflow, resolver=loadingContext.resolver, @@ -1250,7 +1249,7 @@ def main( if args.parallel: temp_executor = MultithreadedJobExecutor() runtimeContext.select_resources = temp_executor.select_resources - real_executor = temp_executor # type: JobExecutor + real_executor: JobExecutor = temp_executor else: real_executor = SingleJobExecutor() else: @@ -1260,7 +1259,7 @@ def main( runtimeContext.basedir = input_basedir if isinstance(tool, ProcessGenerator): - tfjob_order = {} # type: CWLObjectType + tfjob_order: CWLObjectType = {} if loadingContext.jobdefaults: tfjob_order.update(loadingContext.jobdefaults) if job_order_object: @@ -1290,7 +1289,7 @@ def main( if isinstance(err.code, int): return err.code else: - _logger.debug("Non-integer SystemExit: %s", err.code) + _logger.debug("Non-integer SystemExit: %s", err.code, exc_info=args.debug) return 1 del args.workflow @@ -1425,8 +1424,7 @@ def loc_to_path(obj: CWLObjectType) -> None: # public API for logging.StreamHandler prov_log_handler.close() close_ro(research_obj, args.provenance) - - _logger.removeHandler(stderr_handler) + _logger.removeHandler(err_handler) _logger.addHandler(defaultStreamHandler) diff --git a/cwltool/mpi.py b/cwltool/mpi.py index 2cc1122c6..a7bdcbe03 100644 --- a/cwltool/mpi.py +++ b/cwltool/mpi.py @@ -3,7 +3,8 @@ import inspect import os import re -from typing import List, Mapping, MutableMapping, Optional, Type, TypeVar, Union +from collections.abc import Mapping, MutableMapping +from typing import Optional, TypeVar, Union from schema_salad.utils import yaml_no_ts @@ -18,9 +19,9 @@ def __init__( runner: str = "mpirun", nproc_flag: str = "-n", default_nproc: Union[int, str] = 1, - extra_flags: Optional[List[str]] = None, - env_pass: Optional[List[str]] = None, - env_pass_regex: Optional[List[str]] = None, + extra_flags: Optional[list[str]] = None, + env_pass: Optional[list[str]] = None, + env_pass_regex: Optional[list[str]] = None, env_set: Optional[Mapping[str, str]] = None, ) -> None: """ @@ -46,7 +47,7 @@ def __init__( self.env_set = env_set or {} @classmethod - def load(cls: Type[MpiConfigT], config_file_name: str) -> MpiConfigT: + def load(cls: type[MpiConfigT], config_file_name: str) -> MpiConfigT: """Create the MpiConfig object from the contents of a YAML file. The file must contain exactly one object, whose attributes must diff --git a/cwltool/mutation.py b/cwltool/mutation.py index 077b92cb7..622807ec6 100644 --- a/cwltool/mutation.py +++ b/cwltool/mutation.py @@ -1,10 +1,16 @@ -from collections import namedtuple -from typing import Dict, cast +"""Support for InplaceUpdateRequirement.""" + +from typing import NamedTuple, cast from .errors import WorkflowException from .utils import CWLObjectType -MutationState = namedtuple("MutationState", ["generation", "readers", "stepname"]) + +class _MutationState(NamedTuple): + generation: int + readers: list[str] + stepname: str + _generation = "http://commonwl.org/cwltool#generation" @@ -20,11 +26,11 @@ class MutationManager: def __init__(self) -> None: """Initialize.""" - self.generations: Dict[str, MutationState] = {} + self.generations: dict[str, _MutationState] = {} def register_reader(self, stepname: str, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) - current = self.generations.get(loc, MutationState(0, [], "")) + current = self.generations.get(loc, _MutationState(0, [], "")) obj_generation = obj.get(_generation, 0) if obj_generation != current.generation: @@ -40,7 +46,7 @@ def register_reader(self, stepname: str, obj: CWLObjectType) -> None: def release_reader(self, stepname: str, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) - current = self.generations.get(loc, MutationState(0, [], "")) + current = self.generations.get(loc, _MutationState(0, [], "")) obj_generation = obj.get(_generation, 0) if obj_generation != current.generation: @@ -55,7 +61,7 @@ def release_reader(self, stepname: str, obj: CWLObjectType) -> None: def register_mutation(self, stepname: str, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) - current = self.generations.get(loc, MutationState(0, [], "")) + current = self.generations.get(loc, _MutationState(0, [], "")) obj_generation = obj.get(_generation, 0) if len(current.readers) > 0: @@ -73,11 +79,11 @@ def register_mutation(self, stepname: str, obj: CWLObjectType) -> None: ) ) - self.generations[loc] = MutationState(current.generation + 1, current.readers, stepname) + self.generations[loc] = _MutationState(current.generation + 1, current.readers, stepname) def set_generation(self, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) - current = self.generations.get(loc, MutationState(0, [], "")) + current = self.generations.get(loc, _MutationState(0, [], "")) obj[_generation] = current.generation def unset_generation(self, obj: CWLObjectType) -> None: diff --git a/cwltool/pack.py b/cwltool/pack.py index c9fbc4e04..d3705d5e4 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -2,17 +2,8 @@ import copy import urllib -from typing import ( - Any, - Callable, - Dict, - MutableMapping, - MutableSequence, - Optional, - Set, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from typing import Any, Callable, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.ref_resolver import Loader, SubLoader @@ -30,7 +21,7 @@ def find_run( d: Union[CWLObjectType, ResolveType], loadref: LoadRefType, - runs: Set[str], + runs: set[str], ) -> None: if isinstance(d, MutableSequence): for s in d: @@ -46,7 +37,7 @@ def find_run( def find_ids( d: Union[CWLObjectType, CWLOutputType, MutableSequence[CWLObjectType], None], - ids: Set[str], + ids: set[str], ) -> None: if isinstance(d, MutableSequence): for s in d: @@ -59,7 +50,8 @@ def find_ids( find_ids(cast(CWLOutputType, s2), ids) -def replace_refs(d: Any, rewrite: Dict[str, str], stem: str, newstem: str) -> None: +def replace_refs(d: Any, rewrite: dict[str, str], stem: str, newstem: str) -> None: + """Replace references with the actual value.""" if isinstance(d, MutableSequence): for s, v in enumerate(d): if isinstance(v, str): @@ -88,7 +80,7 @@ def replace_refs(d: Any, rewrite: Dict[str, str], stem: str, newstem: str) -> No def import_embed( d: Union[MutableSequence[CWLObjectType], CWLObjectType, CWLOutputType], - seen: Set[str], + seen: set[str], ) -> None: if isinstance(d, MutableSequence): for v in d: @@ -114,7 +106,7 @@ def import_embed( def pack( loadingContext: LoadingContext, uri: str, - rewrite_out: Optional[Dict[str, str]] = None, + rewrite_out: Optional[dict[str, str]] = None, loader: Optional[Loader] = None, ) -> CWLObjectType: # The workflow document we have in memory right now may have been @@ -153,7 +145,7 @@ def pack( document_loader.idx[po["id"]] = CommentedMap(po.items()) document_loader.idx[metadata["id"]] = CommentedMap(metadata.items()) - found_versions = {cast(str, loadingContext.metadata["cwlVersion"])} # type: Set[str] + found_versions: set[str] = {cast(str, loadingContext.metadata["cwlVersion"])} def loadref(base: Optional[str], lr_uri: str) -> ResolveType: lr_loadingContext = loadingContext.copy() @@ -167,15 +159,15 @@ def loadref(base: Optional[str], lr_uri: str) -> ResolveType: raise Exception("loader should not be None") return lr_loadingContext.loader.resolve_ref(lr_uri, base_url=base)[0] - input_ids: Set[str] = set() - output_ids: Set[str] = set() + input_ids: set[str] = set() + output_ids: set[str] = set() if isinstance(processobj, MutableSequence): mainobj = processobj[0] else: mainobj = processobj - find_ids(cast(Dict[str, Any], mainobj)["inputs"], input_ids) - find_ids(cast(Dict[str, Any], mainobj)["outputs"], output_ids) + find_ids(cast(dict[str, Any], mainobj)["inputs"], input_ids) + find_ids(cast(dict[str, Any], mainobj)["outputs"], output_ids) runs = {uri} find_run(processobj, loadref, runs) @@ -190,15 +182,15 @@ def loadref(base: Optional[str], lr_uri: str) -> ResolveType: for f in runs: find_ids(document_loader.resolve_ref(f)[0], input_ids) - input_names: Set[str] = set() - output_names: Set[str] = set() + input_names: set[str] = set() + output_names: set[str] = set() - rewrite_inputs: Dict[str, str] = {} - rewrite_outputs: Dict[str, str] = {} + rewrite_inputs: dict[str, str] = {} + rewrite_outputs: dict[str, str] = {} mainpath, _ = urllib.parse.urldefrag(uri) - def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) -> None: + def rewrite_id(r: str, mainuri: str, rewrite: dict[str, str], names: set[str]) -> None: if r == mainuri: rewrite[r] = "#main" elif r.startswith(mainuri) and r[len(mainuri)] in ("#", "/"): @@ -225,7 +217,7 @@ def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) - packed = CommentedMap((("$graph", CommentedSeq()), ("cwlVersion", update_to_version))) namespaces = metadata.get("$namespaces", None) - schemas: Set[str] = set() + schemas: set[str] = set() if "$schemas" in metadata: for each_schema in metadata["$schemas"]: schemas.add(each_schema) @@ -261,7 +253,7 @@ def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) - "Operation", ): continue - dc = cast(Dict[str, Any], copy.deepcopy(dcr)) + dc = cast(dict[str, Any], copy.deepcopy(dcr)) v = rewrite_inputs[r] dc["id"] = v for n in ("name", "cwlVersion", "$namespaces", "$schemas"): diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 0a06eb47b..774414f0a 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -1,20 +1,10 @@ -import collections import logging import os import stat import urllib import uuid -from typing import ( - Dict, - ItemsView, - Iterable, - Iterator, - KeysView, - List, - Optional, - Tuple, - cast, -) +from collections.abc import ItemsView, Iterable, Iterator, KeysView +from typing import NamedTuple, Optional, cast from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException @@ -25,31 +15,24 @@ from .stdfsaccess import abspath from .utils import CWLObjectType, dedup, downloadHttpFile -MapperEnt = collections.namedtuple("MapperEnt", ["resolved", "target", "type", "staged"]) -""" Mapper entries. -.. py:attribute:: resolved - :type: str +class MapperEnt(NamedTuple): + """Mapper entries.""" - The "real" path on the local file system (after resolving relative paths - and traversing symlinks - -.. py:attribute:: target - :type: str - - The path on the target file system (under stagedir) - -.. py:attribute:: type - :type: str - - The object type. One of "File", "Directory", "CreateFile", "WritableFile", - or "CreateWritableFile". - -.. py:attribute:: staged - :type: bool - - If the File has been staged yet -""" + resolved: str + """ + The "real" path on the local file system (after resolving relative paths + and traversing symlinks + """ + target: str + """The path on the target file system (under stagedir)""" + type: Optional[str] + """ + The object type. One of "File", "Directory", "CreateFile", "WritableFile", + or "CreateWritableFile". + """ + staged: Optional[bool] + """If the File has been staged yet.""" @mypyc_attr(allow_interpreted_subclasses=True) @@ -92,20 +75,20 @@ class PathMapper: def __init__( self, - referenced_files: List[CWLObjectType], + referenced_files: list[CWLObjectType], basedir: str, stagedir: str, separateDirs: bool = True, ) -> None: """Initialize the PathMapper.""" - self._pathmap: Dict[str, MapperEnt] = {} + self._pathmap: dict[str, MapperEnt] = {} self.stagedir = stagedir self.separateDirs = separateDirs self.setup(dedup(referenced_files), basedir) def visitlisting( self, - listing: List[CWLObjectType], + listing: list[CWLObjectType], stagedir: str, basedir: str, copy: bool = False, @@ -147,7 +130,7 @@ def visit( if location.startswith("file://"): staged = False self.visitlisting( - cast(List[CWLObjectType], obj.get("listing", [])), + cast(list[CWLObjectType], obj.get("listing", [])), tgt, basedir, copy=copy, @@ -158,7 +141,7 @@ def visit( ab = abspath(path, basedir) if "contents" in obj and path.startswith("_:"): self._pathmap[path] = MapperEnt( - obj["contents"], + cast(str, obj["contents"]), tgt, "CreateWritableFile" if copy else "CreateFile", staged, @@ -189,16 +172,19 @@ def visit( deref, tgt, "WritableFile" if copy else "File", staged ) self.visitlisting( - cast(List[CWLObjectType], obj.get("secondaryFiles", [])), + cast(list[CWLObjectType], obj.get("secondaryFiles", [])), stagedir, basedir, copy=copy, staged=staged, ) - def setup(self, referenced_files: List[CWLObjectType], basedir: str) -> None: - # Go through each file and set the target to its own directory along - # with any secondary files. + def setup(self, referenced_files: list[CWLObjectType], basedir: str) -> None: + """ + For each file, set the target to its own directory. + + Also processes secondary files into that same directory. + """ stagedir = self.stagedir for fob in referenced_files: if self.separateDirs: @@ -246,15 +232,17 @@ def parents(path: str) -> Iterable[str]: def reversemap( self, target: str, - ) -> Optional[Tuple[str, str]]: + ) -> Optional[tuple[str, str]]: """Find the (source, resolved_path) for the given target, if any.""" for k, v in self._pathmap.items(): if v[1] == target: return (k, v[0]) return None - def update(self, key: str, resolved: str, target: str, ctype: str, stage: bool) -> MapperEnt: - """Update an existine entry.""" + def update( + self, key: str, resolved: str, target: str, ctype: Optional[str], stage: Optional[bool] + ) -> MapperEnt: + """Update an existing entry.""" m = MapperEnt(resolved, target, ctype, stage) self._pathmap[key] = m return m diff --git a/cwltool/process.py b/cwltool/process.py index bde035118..6d1fc7ee7 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -13,25 +13,10 @@ import textwrap import urllib.parse import uuid +from collections.abc import Iterable, Iterator, MutableMapping, MutableSequence, Sized +from importlib.resources import files from os import scandir -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - Iterator, - List, - MutableMapping, - MutableSequence, - Optional, - Set, - Sized, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast from cwl_utils import expression from mypy_extensions import mypyc_attr @@ -70,7 +55,6 @@ aslist, cmp_like_py2, ensure_writable, - files, get_listing, normalizeFilesDirs, random_outdir, @@ -161,14 +145,14 @@ def filter(self, record: logging.LogRecord) -> bool: "vocab_res_proc.yml", ) -SCHEMA_CACHE: Dict[ - str, Tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader] +SCHEMA_CACHE: dict[ + str, tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader] ] = {} SCHEMA_FILE: Optional[CWLObjectType] = None SCHEMA_DIR: Optional[CWLObjectType] = None SCHEMA_ANY: Optional[CWLObjectType] = None -custom_schemas: Dict[str, Tuple[str, str]] = {} +custom_schemas: dict[str, tuple[str, str]] = {} def use_standard_schema(version: str) -> None: @@ -186,11 +170,11 @@ def use_custom_schema(version: str, name: str, text: str) -> None: def get_schema( version: str, -) -> Tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader]: +) -> tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader]: if version in SCHEMA_CACHE: return SCHEMA_CACHE[version] - cache: Dict[str, Union[str, Graph, bool]] = {} + cache: dict[str, Union[str, Graph, bool]] = {} version = version.split("#")[-1] if ".dev" in version: version = ".".join(version.split(".")[:-1]) @@ -244,9 +228,9 @@ def stage_files( :raises WorkflowException: if there is a file staging conflict """ items = pathmapper.items() if not symlink else pathmapper.items_exclude_children() - targets: Dict[str, MapperEnt] = {} + targets: dict[str, MapperEnt] = {} for key, entry in list(items): - if "File" not in entry.type: + if entry.type is None or "File" not in entry.type: continue if entry.target not in targets: targets[entry.target] = entry @@ -309,11 +293,11 @@ def stage_files( def relocateOutputs( outputObj: CWLObjectType, destination_path: str, - source_directories: Set[str], + source_directories: set[str], action: str, fs_access: StdFsAccess, compute_checksum: bool = True, - path_mapper: Type[PathMapper] = PathMapper, + path_mapper: type[PathMapper] = PathMapper, ) -> CWLObjectType: adjustDirObjs(outputObj, functools.partial(get_listing, fs_access, recursive=True)) @@ -321,7 +305,7 @@ def relocateOutputs( return outputObj def _collectDirEntries( - obj: Union[CWLObjectType, MutableSequence[CWLObjectType], None] + obj: Union[CWLObjectType, MutableSequence[CWLObjectType], None], ) -> Iterator[CWLObjectType]: if isinstance(obj, dict): if obj.get("class") in ("File", "Directory"): @@ -414,7 +398,7 @@ def add_sizes(fsaccess: StdFsAccess, obj: CWLObjectType) -> None: def fill_in_defaults( - inputs: List[CWLObjectType], + inputs: list[CWLObjectType], job: CWLObjectType, fsaccess: StdFsAccess, ) -> None: @@ -578,7 +562,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext self.tool["id"] = "_:" + str(uuid.uuid4()) self.requirements.extend( cast( - List[CWLObjectType], + list[CWLObjectType], get_overrides(getdefault(loadingContext.overrides_list, []), self.tool["id"]).get( "requirements", [] ), @@ -617,7 +601,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext avroize_type(cast(MutableSequence[CWLOutputType], sdtypes)) av = make_valid_avro( sdtypes, - {cast(str, t["name"]): cast(Dict[str, Any], t) for t in sdtypes}, + {cast(str, t["name"]): cast(dict[str, Any], t) for t in sdtypes}, set(), vocab=INPUT_OBJ_VOCAB, ) @@ -655,9 +639,9 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext c["type"] = avroize_type(c["type"], c["name"]) if key == "inputs": - cast(List[CWLObjectType], self.inputs_record_schema["fields"]).append(c) + cast(list[CWLObjectType], self.inputs_record_schema["fields"]).append(c) elif key == "outputs": - cast(List[CWLObjectType], self.outputs_record_schema["fields"]).append(c) + cast(list[CWLObjectType], self.outputs_record_schema["fields"]).append(c) with SourceLine(toolpath_object, "inputs", ValidationException, debug): self.inputs_record_schema = cast( @@ -681,7 +665,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext if toolpath_object.get("class") is not None and not getdefault( loadingContext.disable_js_validation, False ): - validate_js_options: Optional[Dict[str, Union[List[str], str, int]]] = None + validate_js_options: Optional[dict[str, Union[list[str], str, int]]] = None if loadingContext.js_hint_options_file is not None: try: with open(loadingContext.js_hint_options_file) as options_file: @@ -784,7 +768,7 @@ def _init_job(self, joborder: CWLObjectType, runtime_context: RuntimeContext) -> v = job[k] dircount = [0] - def inc(d: List[int]) -> None: + def inc(d: list[int]) -> None: d[0] += 1 visit_class(v, ("Directory",), lambda x: inc(dircount)) # noqa: B023 @@ -820,18 +804,53 @@ def inc(d: List[int]) -> None: except (ValidationException, WorkflowException) as err: raise WorkflowException("Invalid job input record:\n" + str(err)) from err - files: List[CWLObjectType] = [] + files: list[CWLObjectType] = [] bindings = CommentedSeq() outdir = "" tmpdir = "" stagedir = "" - docker_req, _ = self.get_requirement("DockerRequirement") + docker_req, docker_required = self.get_requirement("DockerRequirement") default_docker = None + mpi_req, mpi_required = self.get_requirement(MPIRequirementName) if docker_req is None and runtime_context.default_container: default_docker = runtime_context.default_container + if ( + docker_req is not None + and runtime_context.use_container + and not runtime_context.singularity + and not runtime_context.user_space_docker_cmd + and mpi_req is not None + ): + if mpi_required: + if docker_required: + raise UnsupportedRequirement( + "No support for DockerRequirement and MPIRequirement " + "both being required, unless Singularity or uDocker is being used." + ) + else: + _logger.warning( + "MPI has been required while DockerRequirement is hinted " + "and neither Singularity nor uDocker is being used, discarding Docker hint(s)." + ) + self.hints = [h for h in self.hints if h["class"] != "DockerRequirement"] + docker_req = None + docker_required = False + else: + if docker_required: + _logger.warning( + "Docker has been required (and neither Singularity nor " + "uDocker is being used) while MPI is hinted, discarding MPI hint(s)/" + ) + self.hints = [h for h in self.hints if h["class"] != MPIRequirementName] + else: + raise UnsupportedRequirement( + "Both Docker and MPI have been hinted and neither " + "Singularity nor uDocker are being used - don't know what to do." + ) + if (docker_req or default_docker) and runtime_context.use_container: if docker_req is not None: # Check if docker output directory is absolute @@ -947,7 +966,7 @@ def inc(d: List[int]) -> None: def evalResources( self, builder: Builder, runtimeContext: RuntimeContext - ) -> Dict[str, Union[int, float]]: + ) -> dict[str, Union[int, float]]: resourceReq, _ = self.get_requirement("ResourceRequirement") if resourceReq is None: resourceReq = {} @@ -957,7 +976,7 @@ def evalResources( ram = 1024 else: ram = 256 - request: Dict[str, Union[int, float, str]] = { + request: dict[str, Union[int, float, str]] = { "coresMin": 1, "coresMax": 1, "ramMin": ram, @@ -1005,7 +1024,7 @@ def evalResources( request[a + "Min"] = mn request[a + "Max"] = cast(Union[int, float], mx) - request_evaluated = cast(Dict[str, Union[int, float]], request) + request_evaluated = cast(dict[str, Union[int, float]], request) if runtimeContext.select_resources is not None: # Call select resources hook return runtimeContext.select_resources(request_evaluated, runtimeContext) @@ -1038,7 +1057,7 @@ def checkRequirements( f"Unsupported requirement {entry['class']}." ) - def validate_hints(self, avsc_names: Names, hints: List[CWLObjectType], strict: bool) -> None: + def validate_hints(self, avsc_names: Names, hints: list[CWLObjectType], strict: bool) -> None: """Process the hints field.""" if self.doc_loader is None: return @@ -1085,10 +1104,11 @@ def __str__(self) -> str: return f"{type(self).__name__}: {self.tool['id']}" -_names: Set[str] = set() +_names: set[str] = set() -def uniquename(stem: str, names: Optional[Set[str]] = None) -> str: +def uniquename(stem: str, names: Optional[set[str]] = None) -> str: + """Construct a thread-unique name using the given stem as a prefix.""" global _names if names is None: names = _names @@ -1123,8 +1143,8 @@ def nestdir(base: str, deps: CWLObjectType) -> CWLObjectType: def mergedirs( listing: MutableSequence[CWLObjectType], ) -> MutableSequence[CWLObjectType]: - r: List[CWLObjectType] = [] - ents: Dict[str, CWLObjectType] = {} + r: list[CWLObjectType] = [] + ents: dict[str, CWLObjectType] = {} for e in listing: basename = cast(str, e["basename"]) if basename not in ents: @@ -1138,14 +1158,14 @@ def mergedirs( if e.get("listing"): # name already in entries # merge it into the existing listing - cast(List[CWLObjectType], ents[basename].setdefault("listing", [])).extend( - cast(List[CWLObjectType], e["listing"]) + cast(list[CWLObjectType], ents[basename].setdefault("listing", [])).extend( + cast(list[CWLObjectType], e["listing"]) ) for e in ents.values(): if e["class"] == "Directory" and "listing" in e: e["listing"] = cast( MutableSequence[CWLOutputType], - mergedirs(cast(List[CWLObjectType], e["listing"])), + mergedirs(cast(list[CWLObjectType], e["listing"])), ) r.extend(ents.values()) return r @@ -1157,8 +1177,8 @@ def mergedirs( def scandeps( base: str, doc: Union[CWLObjectType, MutableSequence[CWLObjectType]], - reffields: Set[str], - urlfields: Set[str], + reffields: set[str], + urlfields: set[str], loadref: Callable[[str, str], Union[CommentedMap, CommentedSeq, str, None]], urljoin: Callable[[str, str], str] = urllib.parse.urljoin, nestdirs: bool = True, diff --git a/cwltool/procgenerator.py b/cwltool/procgenerator.py index 34c1e650f..07123f906 100644 --- a/cwltool/procgenerator.py +++ b/cwltool/procgenerator.py @@ -1,5 +1,5 @@ import copy -from typing import Dict, Optional, Tuple, cast +from typing import Optional, cast from ruamel.yaml.comments import CommentedMap from schema_salad.exceptions import ValidationException @@ -57,7 +57,7 @@ def job( except WorkflowException: raise except Exception as exc: - _logger.exception("Unexpected exception") + _logger.exception("Unexpected exception", exc_info=runtimeContext.debug) raise WorkflowException(str(exc)) from exc @@ -80,7 +80,7 @@ def __init__( self.embedded_tool = load_tool(toolpath_object["run"], loadingContext) except ValidationException as vexc: if loadingContext.debug: - _logger.exception("Validation exception") + _logger.exception("Validation exception", exc_info=loadingContext.debug) raise WorkflowException( "Tool definition %s failed validation:\n%s" % (toolpath_object["run"], indent(str(vexc))) @@ -99,16 +99,16 @@ def result( job_order: CWLObjectType, jobout: CWLObjectType, runtimeContext: RuntimeContext, - ) -> Tuple[Process, CWLObjectType]: + ) -> tuple[Process, CWLObjectType]: try: loadingContext = self.loadingContext.copy() loadingContext.metadata = {} embedded_tool = load_tool( - cast(Dict[str, str], jobout["runProcess"])["location"], loadingContext + cast(dict[str, str], jobout["runProcess"])["location"], loadingContext ) except ValidationException as vexc: if runtimeContext.debug: - _logger.exception("Validation exception") + _logger.exception("Validation exception", exc_info=runtimeContext.debug) raise WorkflowException( "Tool definition %s failed validation:\n%s" % (jobout["runProcess"], indent(str(vexc))) diff --git a/cwltool/resolver.py b/cwltool/resolver.py index e48957f26..918a9b24e 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -15,8 +15,8 @@ def resolve_local(document_loader: Optional[Loader], uri: str) -> Optional[str]: try: pathobj = Path(pathpart).resolve() - except OSError: - _logger.debug("local resolver could not resolve %s", uri) + except OSError as exc: + _logger.debug("local resolver could not resolve %s due to %s", uri, str(exc)) return None if pathobj.is_file(): diff --git a/cwltool/run_job.py b/cwltool/run_job.py index 307872f7a..5a81ce20c 100644 --- a/cwltool/run_job.py +++ b/cwltool/run_job.py @@ -4,10 +4,10 @@ import os import subprocess # nosec import sys -from typing import BinaryIO, Dict, List, Optional, TextIO, Union +from typing import BinaryIO, Optional, TextIO, Union -def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[str, str]: +def handle_software_environment(cwl_env: dict[str, str], script: str) -> dict[str, str]: """Update the provided environment dict by running the script.""" exec_env = cwl_env.copy() exec_env["_CWLTOOL"] = "1" @@ -29,7 +29,7 @@ def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[st return env -def main(argv: List[str]) -> int: +def main(argv: list[str]) -> int: """ Read in the configuration JSON and execute the commands. diff --git a/cwltool/schemas/v1.1.0-dev1/cwl-runner.cwl b/cwltool/schemas/v1.1.0-dev1/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.1/cwl-runner.cwl b/cwltool/schemas/v1.1/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev1/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev1/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev2/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev2/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev3/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev3/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev4/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev4/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.2.0-dev5/cwl-runner.cwl b/cwltool/schemas/v1.2.0-dev5/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/schemas/v1.3.0-dev1/cwl-runner.cwl b/cwltool/schemas/v1.3.0-dev1/cwl-runner.cwl old mode 100644 new mode 100755 diff --git a/cwltool/secrets.py b/cwltool/secrets.py index f35f24c37..c73e0108c 100644 --- a/cwltool/secrets.py +++ b/cwltool/secrets.py @@ -1,7 +1,8 @@ """Minimal in memory storage of secrets.""" import uuid -from typing import Dict, List, MutableMapping, MutableSequence, Optional, cast +from collections.abc import MutableMapping, MutableSequence +from typing import Optional, cast from .utils import CWLObjectType, CWLOutputType @@ -11,7 +12,7 @@ class SecretStore: def __init__(self) -> None: """Initialize the secret store.""" - self.secrets: Dict[str, str] = {} + self.secrets: dict[str, str] = {} def add(self, value: Optional[CWLOutputType]) -> Optional[CWLOutputType]: """ @@ -28,7 +29,7 @@ def add(self, value: Optional[CWLOutputType]) -> Optional[CWLOutputType]: return placeholder return value - def store(self, secrets: List[str], job: CWLObjectType) -> None: + def store(self, secrets: list[str], job: CWLObjectType) -> None: """Sanitize the job object of any of the given secrets.""" for j in job: if j in secrets: diff --git a/cwltool/singularity.py b/cwltool/singularity.py index c43183ac7..d0e46fb27 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -6,8 +6,9 @@ import re import shutil import sys +from collections.abc import MutableMapping from subprocess import check_call, check_output # nosec -from typing import Callable, Dict, List, MutableMapping, Optional, Tuple, cast +from typing import Callable, Optional, cast from schema_salad.sourceline import SourceLine from spython.main import Client @@ -28,13 +29,13 @@ # This is a list containing major and minor versions as integer. # (The number of minor version digits can vary among different distributions, # therefore we need a list here.) -_SINGULARITY_VERSION: Optional[List[int]] = None +_SINGULARITY_VERSION: Optional[list[int]] = None # Cached flavor / distribution of singularity # Can be singularity, singularity-ce or apptainer _SINGULARITY_FLAVOR: str = "" -def get_version() -> Tuple[List[int], str]: +def get_version() -> tuple[list[int], str]: """ Parse the output of 'singularity --version' to determine the flavor and version. @@ -74,6 +75,14 @@ def is_apptainer_1_or_newer() -> bool: return v[0][0] >= 1 +def is_apptainer_1_1_or_newer() -> bool: + """Check if apptainer singularity distribution is version 1.1 or higher.""" + v = get_version() + if v[1] != "apptainer": + return False + return v[0][0] >= 2 or (v[0][0] >= 1 and v[0][1] >= 1) + + def is_version_2_6() -> bool: """ Check if this singularity version is exactly version 2.6. @@ -118,11 +127,21 @@ def is_version_3_9_or_newer() -> bool: return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 9) +def is_version_3_10_or_newer() -> bool: + """Detect if Singularity v3.10+ is available.""" + v = get_version() + return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 10) + + def _normalize_image_id(string: str) -> str: + if ":" not in string: + string += "_latest" return string.replace("/", "_") + ".img" def _normalize_sif_id(string: str) -> str: + if ":" not in string: + string += "_latest" return string.replace("/", "_") + ".sif" @@ -131,9 +150,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Builder for invoking the Singularty software container engine.""" @@ -141,7 +160,7 @@ def __init__( @staticmethod def get_image( - dockerRequirement: Dict[str, str], + dockerRequirement: dict[str, str], pull_image: bool, tmp_outdir_prefix: str, force_pull: bool = False, @@ -247,7 +266,7 @@ def get_image( dockerRequirement["dockerImageId"] = path found = True if (force_pull or not found) and pull_image: - cmd = [] # type: List[str] + cmd: list[str] = [] if "dockerPull" in dockerRequirement: if cache_folder: env = os.environ.copy() @@ -338,7 +357,7 @@ def get_from_requirements( if not bool(shutil.which("singularity")): raise WorkflowException("singularity executable is not available") - if not self.get_image(cast(Dict[str, str], r), pull_image, tmp_outdir_prefix, force_pull): + if not self.get_image(cast(dict[str, str], r), pull_image, tmp_outdir_prefix, force_pull): raise WorkflowException("Container image {} not found".format(r["dockerImageId"])) if "CWL_SINGULARITY_CACHE" in os.environ: @@ -350,7 +369,7 @@ def get_from_requirements( return os.path.abspath(img_path) @staticmethod - def append_volume(runtime: List[str], source: str, target: str, writable: bool = False) -> None: + def append_volume(runtime: list[str], source: str, target: str, writable: bool = False) -> None: """Add binding arguments to the runtime list.""" if is_version_3_9_or_newer(): DockerCommandLineJob.append_volume(runtime, source, target, writable, skip_mkdirs=True) @@ -364,7 +383,7 @@ def append_volume(runtime: List[str], source: str, target: str, writable: bool = runtime.append(vol) def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] ) -> None: if not volume.resolved.startswith("_:"): if host_outdir_tgt is not None and not is_version_3_4_or_newer(): @@ -380,7 +399,7 @@ def add_file_or_directory_volume( def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -417,7 +436,7 @@ def add_writable_file_volume( def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -452,7 +471,7 @@ def add_writable_directory_volume( shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt or new_dir) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: return { "TMPDIR": self.CONTAINER_TMPDIR, "HOME": self.builder.outdir, @@ -460,17 +479,21 @@ def _required_env(self) -> Dict[str, str]: def create_runtime( self, env: MutableMapping[str, str], runtime_context: RuntimeContext - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], Optional[str]]: """Return the Singularity runtime list of commands and options.""" any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False + runtime = [ "singularity", "--quiet", - "exec", + "run" if is_apptainer_1_1_or_newer() or is_version_3_10_or_newer() else "exec", "--contain", "--ipc", "--cleanenv", ] + if is_apptainer_1_1_or_newer() or is_version_3_10_or_newer(): + runtime.append("--no-eval") + if singularity_supports_userns(): runtime.append("--userns") else: diff --git a/cwltool/singularity_utils.py b/cwltool/singularity_utils.py index e4cc88918..13f7ed3f6 100644 --- a/cwltool/singularity_utils.py +++ b/cwltool/singularity_utils.py @@ -2,7 +2,7 @@ import os import os.path -from subprocess import DEVNULL, PIPE, Popen, TimeoutExpired # nosec +import subprocess # nosec from typing import Optional _USERNS: Optional[bool] = None @@ -14,17 +14,17 @@ def singularity_supports_userns() -> bool: if _USERNS is None: try: hello_image = os.path.join(os.path.dirname(__file__), "hello.simg") - result = Popen( # nosec + result = subprocess.run( # nosec ["singularity", "exec", "--userns", hello_image, "true"], - stderr=PIPE, - stdout=DEVNULL, - universal_newlines=True, - ).communicate(timeout=60)[1] + capture_output=True, + timeout=60, + text=True, + ).stderr _USERNS = ( "No valid /bin/sh" in result or "/bin/sh doesn't exist in container" in result or "executable file not found in" in result ) - except TimeoutExpired: + except subprocess.TimeoutExpired: _USERNS = False return _USERNS diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index ec99bda05..3d4d48f6b 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -10,17 +10,8 @@ import argparse import os import string -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from typing import TYPE_CHECKING, Any, Optional, Union, cast from .utils import HasReqsHints @@ -59,6 +50,8 @@ class DependenciesConfiguration: def __init__(self, args: argparse.Namespace) -> None: """Initialize.""" + self.tool_dependency_dir: Optional[str] = None + self.dependency_resolvers_config_file: Optional[str] = None conf_file = getattr(args, "beta_dependency_resolvers_configuration", None) tool_dependency_dir = getattr(args, "beta_dependencies_directory", None) conda_dependencies = getattr(args, "beta_conda_dependencies", None) @@ -79,7 +72,8 @@ def __init__(self, args: argparse.Namespace) -> None: if self.tool_dependency_dir and not os.path.exists(self.tool_dependency_dir): os.makedirs(self.tool_dependency_dir) - def build_job_script(self, builder: "Builder", command: List[str]) -> str: + def build_job_script(self, builder: "Builder", command: list[str]) -> str: + """Use the galaxy-tool-util library to construct a build script.""" ensure_galaxy_lib_available() resolution_config_dict = { "use": self.use_tool_dependencies, @@ -103,14 +97,14 @@ def build_job_script(self, builder: "Builder", command: List[str]) -> str: ) ) - template_kwds: Dict[str, str] = dict(handle_dependencies=handle_dependencies) + template_kwds: dict[str, str] = dict(handle_dependencies=handle_dependencies) job_script = COMMAND_WITH_DEPENDENCIES_TEMPLATE.substitute(template_kwds) return job_script def get_dependencies(builder: HasReqsHints) -> ToolRequirements: (software_requirement, _) = builder.get_requirement("SoftwareRequirement") - dependencies: List[Union["ToolRequirement", Dict[str, Any]]] = [] + dependencies: list[Union["ToolRequirement", dict[str, Any]]] = [] if software_requirement and software_requirement.get("packages"): packages = cast( MutableSequence[MutableMapping[str, Union[str, MutableSequence[str]]]], diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index 069289111..c58257f63 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -3,7 +3,7 @@ import glob import os import urllib -from typing import IO, Any, List +from typing import IO, Any from schema_salad.ref_resolver import file_uri, uri_file_path @@ -31,7 +31,8 @@ def __init__(self, basedir: str) -> None: def _abs(self, p: str) -> str: return abspath(p, self.basedir) - def glob(self, pattern: str) -> List[str]: + def glob(self, pattern: str) -> list[str]: + """Return a possibly empty list of absolute URI paths that match pathname.""" return [file_uri(str(self._abs(line))) for line in glob.glob(self._abs(pattern))] def open(self, fn: str, mode: str) -> IO[Any]: @@ -49,7 +50,8 @@ def isfile(self, fn: str) -> bool: def isdir(self, fn: str) -> bool: return os.path.isdir(self._abs(fn)) - def listdir(self, fn: str) -> List[str]: + def listdir(self, fn: str) -> list[str]: + """Return a list containing the absolute path URLs of the entries in the directory given by path.""" return [abspath(urllib.parse.quote(entry), fn) for entry in os.listdir(self._abs(fn))] def join(self, path, *paths): # type: (str, *str) -> str diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index f6df7e69f..0a0289cc3 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -1,18 +1,6 @@ import urllib -from collections import namedtuple -from typing import ( - Any, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from collections.abc import Mapping, MutableMapping, MutableSequence +from typing import Any, NamedTuple, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -22,7 +10,13 @@ from .utils import CWLObjectType, aslist from .workflow import Workflow, WorkflowStep -Node = namedtuple("Node", ("up", "down", "type")) + +class _Node(NamedTuple): + up: list[str] + down: list[str] + type: Optional[str] + + UP = "up" DOWN = "down" INPUT = "input" @@ -30,10 +24,10 @@ STEP = "step" -def subgraph_visit( +def _subgraph_visit( current: str, - nodes: MutableMapping[str, Node], - visited: Set[str], + nodes: MutableMapping[str, _Node], + visited: set[str], direction: str, ) -> None: if current in visited: @@ -45,22 +39,28 @@ def subgraph_visit( if direction == UP: d = nodes[current].up for c in d: - subgraph_visit(c, nodes, visited, direction) + _subgraph_visit(c, nodes, visited, direction) + +def _declare_node(nodes: dict[str, _Node], nodeid: str, tp: Optional[str]) -> _Node: + """ + Record the given nodeid in the graph. -def declare_node(nodes: Dict[str, Node], nodeid: str, tp: Optional[str]) -> Node: + If the nodeid is already present, but its type is unset, set it. + :returns: The Node tuple (even if already present in the graph). + """ if nodeid in nodes: n = nodes[nodeid] if n.type is None: - nodes[nodeid] = Node(n.up, n.down, tp) + nodes[nodeid] = _Node(n.up, n.down, tp) else: - nodes[nodeid] = Node([], [], tp) + nodes[nodeid] = _Node([], [], tp) return nodes[nodeid] def find_step( - steps: List[WorkflowStep], stepid: str, loading_context: LoadingContext -) -> Tuple[Optional[CWLObjectType], Optional[WorkflowStep]]: + steps: list[WorkflowStep], stepid: str, loading_context: LoadingContext +) -> tuple[Optional[CWLObjectType], Optional[WorkflowStep]]: """Find the step (raw dictionary and WorkflowStep) for a given step id.""" for st in steps: st_tool_id = st.tool["id"] @@ -114,22 +114,22 @@ def get_subgraph( if tool.tool["class"] != "Workflow": raise Exception("Can only extract subgraph from workflow") - nodes: Dict[str, Node] = {} + nodes: dict[str, _Node] = {} for inp in tool.tool["inputs"]: - declare_node(nodes, inp["id"], INPUT) + _declare_node(nodes, inp["id"], INPUT) for out in tool.tool["outputs"]: - declare_node(nodes, out["id"], OUTPUT) + _declare_node(nodes, out["id"], OUTPUT) for i in aslist(out.get("outputSource", CommentedSeq)): # source is upstream from output (dependency) nodes[out["id"]].up.append(i) # output is downstream from source - declare_node(nodes, i, None) + _declare_node(nodes, i, None) nodes[i].down.append(out["id"]) for st in tool.tool["steps"]: - step = declare_node(nodes, st["id"], STEP) + step = _declare_node(nodes, st["id"], STEP) for i in st["in"]: if "source" not in i: continue @@ -137,7 +137,7 @@ def get_subgraph( # source is upstream from step (dependency) step.up.append(src) # step is downstream from source - declare_node(nodes, src, None) + _declare_node(nodes, src, None) nodes[src].down.append(st["id"]) for out in st["out"]: if isinstance(out, Mapping) and "id" in out: @@ -145,20 +145,20 @@ def get_subgraph( # output is downstream from step step.down.append(out) # step is upstream from output - declare_node(nodes, out, None) + _declare_node(nodes, out, None) nodes[out].up.append(st["id"]) # Find all the downstream nodes from the starting points - visited_down: Set[str] = set() + visited_down: set[str] = set() for r in roots: if nodes[r].type == OUTPUT: - subgraph_visit(r, nodes, visited_down, UP) + _subgraph_visit(r, nodes, visited_down, UP) else: - subgraph_visit(r, nodes, visited_down, DOWN) + _subgraph_visit(r, nodes, visited_down, DOWN) # Now make sure all the nodes are connected to upstream inputs - visited: Set[str] = set() - rewire: Dict[str, Tuple[str, CWLObjectType]] = {} + visited: set[str] = set() + rewire: dict[str, tuple[str, CWLObjectType]] = {} for v in visited_down: visited.add(v) if nodes[v].type in (STEP, OUTPUT): @@ -221,7 +221,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C extracted["inputs"] = CommentedSeq() extracted["outputs"] = CommentedSeq() - for in_port in cast(List[CWLObjectType], step["in"]): + for in_port in cast(list[CWLObjectType], step["in"]): name = "#" + cast(str, in_port["id"]).split("#")[-1].split("/")[-1] inp: CWLObjectType = {"id": name, "type": "Any"} if "default" in in_port: @@ -231,7 +231,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C if "linkMerge" in in_port: del in_port["linkMerge"] - for outport in cast(List[Union[str, Mapping[str, Any]]], step["out"]): + for outport in cast(list[Union[str, Mapping[str, Any]]], step["out"]): if isinstance(outport, Mapping): outport_id = cast(str, outport["id"]) else: @@ -256,7 +256,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C def get_process( tool: Workflow, step_id: str, loading_context: LoadingContext -) -> Tuple[Any, WorkflowStep]: +) -> tuple[Any, WorkflowStep]: """Find the underlying Process for a given Workflow step id.""" if loading_context.loader is None: raise Exception("loading_context.loader cannot be None") diff --git a/cwltool/udocker.py b/cwltool/udocker.py index 6598d6a7c..ea3fc78ca 100644 --- a/cwltool/udocker.py +++ b/cwltool/udocker.py @@ -1,7 +1,5 @@ """Enables Docker software containers via the udocker runtime.""" -from typing import List - from .docker import DockerCommandLineJob @@ -10,7 +8,7 @@ class UDockerCommandLineJob(DockerCommandLineJob): @staticmethod def append_volume( - runtime: List[str], + runtime: list[str], source: str, target: str, writable: bool = False, diff --git a/cwltool/update.py b/cwltool/update.py index 4fd66b37a..67e1f4257 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -1,15 +1,7 @@ import copy +from collections.abc import MutableMapping, MutableSequence from functools import partial -from typing import ( - Callable, - Dict, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from typing import Callable, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException @@ -20,7 +12,7 @@ from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field -def v1_2to1_3dev1(doc: CommentedMap, loader: Loader, baseuri: str) -> Tuple[CommentedMap, str]: +def v1_2to1_3dev1(doc: CommentedMap, loader: Loader, baseuri: str) -> tuple[CommentedMap, str]: """Public updater for v1.2 to v1.3.0-dev1.""" doc = copy.deepcopy(doc) @@ -78,7 +70,7 @@ def rewrite_loop_requirements(t: CWLObjectType) -> None: def v1_1to1_2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.1 to v1.2.""" doc = copy.deepcopy(doc) @@ -94,7 +86,7 @@ def v1_1to1_2( def v1_0to1_1( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.0 to v1.1.""" doc = copy.deepcopy(doc) @@ -195,21 +187,21 @@ def fix_inputBinding(t: CWLObjectType) -> None: def v1_1_0dev1to1_1( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.1.0-dev1 to v1.1.""" return (doc, "v1.1") def v1_2_0dev1todev2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev1 to v1.2.0-dev2.""" return (doc, "v1.2.0-dev2") def v1_2_0dev2todev3( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev2 to v1.2.0-dev3.""" doc = copy.deepcopy(doc) @@ -232,21 +224,21 @@ def update_pickvalue(t: CWLObjectType) -> None: def v1_2_0dev3todev4( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev3 to v1.2.0-dev4.""" return (doc, "v1.2.0-dev4") def v1_2_0dev4todev5( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev4 to v1.2.0-dev5.""" return (doc, "v1.2.0-dev5") def v1_2_0dev5to1_2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev5 to v1.2.""" return (doc, "v1.2") @@ -264,13 +256,13 @@ def v1_2_0dev5to1_2( "v1.3.0-dev1", ] -UPDATES: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] = { +UPDATES: dict[str, Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]]] = { "v1.0": v1_0to1_1, "v1.1": v1_1to1_2, "v1.2": v1_2to1_3dev1, } -DEVUPDATES: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] = { +DEVUPDATES: dict[str, Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]]] = { "v1.1.0-dev1": v1_1_0dev1to1_1, "v1.2.0-dev1": v1_2_0dev1todev2, "v1.2.0-dev2": v1_2_0dev2todev3, @@ -291,7 +283,7 @@ def v1_2_0dev5to1_2( def identity( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Do-nothing, CWL document upgrade function.""" return (doc, cast(str, doc["cwlVersion"])) @@ -300,7 +292,7 @@ def checkversion( doc: Union[CommentedSeq, CommentedMap], metadata: CommentedMap, enable_dev: bool, -) -> Tuple[CommentedMap, str]: +) -> tuple[CommentedMap, str]: """Check the validity of the version of the give CWL document. Returns the document and the validated version string. @@ -365,7 +357,7 @@ def update( (cdoc, version) = checkversion(doc, metadata, enable_dev) originalversion = copy.copy(version) - nextupdate: Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]] = identity + nextupdate: Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]] = identity while version != update_to and nextupdate: (cdoc, version) = nextupdate(cdoc, loader, baseuri) diff --git a/cwltool/utils.py b/cwltool/utils.py index c8620994a..c37dbe8e7 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -19,6 +19,7 @@ import tempfile import urllib import uuid +from collections.abc import Generator, Iterable, MutableMapping, MutableSequence from datetime import datetime from email.utils import parsedate_to_datetime from functools import partial @@ -31,17 +32,9 @@ Any, Callable, Deque, - Dict, - Generator, - Iterable, - List, Literal, - MutableMapping, - MutableSequence, NamedTuple, Optional, - Set, - Tuple, TypedDict, Union, cast, @@ -54,19 +47,12 @@ from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader -if sys.version_info >= (3, 9): - from importlib.resources import as_file, files -else: - from importlib_resources import as_file, files - if TYPE_CHECKING: from .command_line_tool import CallbackJob, ExpressionJob from .job import CommandLineJob, JobBase from .stdfsaccess import StdFsAccess from .workflow_job import WorkflowJob -__all__ = ["files", "as_file"] - __random_outdir: Optional[str] = None CONTENT_LIMIT = 64 * 1024 @@ -92,13 +78,13 @@ OutputCallbackType = Callable[[Optional[CWLObjectType], str], None] ResolverType = Callable[["Loader", str], Optional[str]] DestinationsType = MutableMapping[str, Optional[CWLOutputType]] -ScatterDestinationsType = MutableMapping[str, List[Optional[CWLOutputType]]] +ScatterDestinationsType = MutableMapping[str, list[Optional[CWLOutputType]]] ScatterOutputCallbackType = Callable[[Optional[ScatterDestinationsType], str], None] SinkType = Union[CWLOutputType, CWLObjectType] DirectoryType = TypedDict( - "DirectoryType", {"class": str, "listing": List[CWLObjectType], "basename": str} + "DirectoryType", {"class": str, "listing": list[CWLObjectType], "basename": str} ) -JSONType = Union[Dict[str, "JSONType"], List["JSONType"], str, int, float, bool, None] +JSONType = Union[dict[str, "JSONType"], list["JSONType"], str, int, float, bool, None] class WorkflowStateItem(NamedTuple): @@ -109,7 +95,7 @@ class WorkflowStateItem(NamedTuple): success: str -ParametersType = List[CWLObjectType] +ParametersType = list[CWLObjectType] StepType = CWLObjectType # WorkflowStep LoadListingType = Union[Literal["no_listing"], Literal["shallow_listing"], Literal["deep_listing"]] @@ -143,7 +129,7 @@ def copytree_with_merge(src: str, dst: str) -> None: shutil.copy2(spath, dpath) -def cmp_like_py2(dict1: Dict[str, Any], dict2: Dict[str, Any]) -> int: +def cmp_like_py2(dict1: dict[str, Any], dict2: dict[str, Any]) -> int: """ Compare in the same manner as Python2. @@ -259,20 +245,20 @@ def adjustDirObjs(rec: Any, op: Union[Callable[..., Any], "partial[Any]"]) -> No visit_class(rec, ("Directory",), op) -def dedup(listing: List[CWLObjectType]) -> List[CWLObjectType]: +def dedup(listing: list[CWLObjectType]) -> list[CWLObjectType]: marksub = set() - def mark(d: Dict[str, str]) -> None: + def mark(d: dict[str, str]) -> None: marksub.add(d["location"]) for entry in listing: if entry["class"] == "Directory": - for e in cast(List[CWLObjectType], entry.get("listing", [])): + for e in cast(list[CWLObjectType], entry.get("listing", [])): adjustFileObjs(e, mark) adjustDirObjs(e, mark) dd = [] - markdup: Set[str] = set() + markdup: set[str] = set() for r in listing: if r["location"] not in marksub and r["location"] not in markdup: dd.append(r) @@ -284,14 +270,14 @@ def mark(d: Dict[str, str]) -> None: def get_listing(fs_access: "StdFsAccess", rec: CWLObjectType, recursive: bool = True) -> None: """Expand, recursively, any 'listing' fields in a Directory.""" if rec.get("class") != "Directory": - finddirs: List[CWLObjectType] = [] + finddirs: list[CWLObjectType] = [] visit_class(rec, ("Directory",), finddirs.append) for f in finddirs: get_listing(fs_access, f, recursive=recursive) return if "listing" in rec: return - listing: List[CWLOutputType] = [] + listing: list[CWLOutputType] = [] loc = cast(str, rec["location"]) for ld in fs_access.listdir(loc): parse = urllib.parse.urlparse(ld) @@ -310,7 +296,7 @@ def get_listing(fs_access: "StdFsAccess", rec: CWLObjectType, recursive: bool = rec["listing"] = listing -def trim_listing(obj: Dict[str, Any]) -> None: +def trim_listing(obj: dict[str, Any]) -> None: """ Remove 'listing' field from Directory objects that are file references. @@ -322,7 +308,7 @@ def trim_listing(obj: Dict[str, Any]) -> None: del obj["listing"] -def downloadHttpFile(httpurl: str) -> Tuple[str, Optional[datetime]]: +def downloadHttpFile(httpurl: str) -> tuple[str, Optional[datetime]]: """ Download a remote file, possibly using a locally cached copy. @@ -412,9 +398,9 @@ def normalizeFilesDirs( MutableMapping[str, Any], DirectoryType, ] - ] + ], ) -> None: - def addLocation(d: Dict[str, Any]) -> None: + def addLocation(d: dict[str, Any]) -> None: if "location" not in d: if d["class"] == "File" and ("contents" not in d): raise ValidationException( @@ -484,10 +470,10 @@ class HasReqsHints: def __init__(self) -> None: """Initialize this reqs decorator.""" - self.requirements: List[CWLObjectType] = [] - self.hints: List[CWLObjectType] = [] + self.requirements: list[CWLObjectType] = [] + self.hints: list[CWLObjectType] = [] - def get_requirement(self, feature: str) -> Tuple[Optional[CWLObjectType], Optional[bool]]: + def get_requirement(self, feature: str) -> tuple[Optional[CWLObjectType], Optional[bool]]: """Retrieve the named feature from the requirements field, or the hints field.""" for item in reversed(self.requirements): if item["class"] == feature: diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index de4adaa14..3a490b68d 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -2,18 +2,9 @@ import itertools import json import logging -from collections import namedtuple -from typing import ( - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from importlib.resources import files +from typing import Any, NamedTuple, Optional, Union, cast from cwl_utils.errors import SubstitutionError from cwl_utils.expression import scanner as scan_expression @@ -32,7 +23,6 @@ from .errors import WorkflowException from .loghandler import _logger -from .utils import files def is_expression(tool: Any, schema: Optional[Schema]) -> bool: @@ -63,7 +53,7 @@ def get_expressions( tool: Union[CommentedMap, str, CommentedSeq], schema: Optional[Union[Schema, ArraySchema]], source_line: Optional[SourceLine] = None, -) -> List[Tuple[str, Optional[SourceLine]]]: +) -> list[tuple[str, Optional[SourceLine]]]: debug = _logger.isEnabledFor(logging.DEBUG) if is_expression(tool, schema): return [(cast(str, tool), source_line)] @@ -119,13 +109,17 @@ def get_expressions( return [] -JSHintJSReturn = namedtuple("JSHintJSReturn", ["errors", "globals"]) +class JSHintJSReturn(NamedTuple): + """List of errors and the final values of the globals from running javascript.""" + + errors: list[str] + globals: list[str] def jshint_js( js_text: str, - globals: Optional[List[str]] = None, - options: Optional[Dict[str, Union[List[str], str, int]]] = None, + globals: Optional[list[str]] = None, + options: Optional[dict[str, Union[list[str], str, int]]] = None, container_engine: str = "docker", eval_timeout: float = 60, ) -> JSHintJSReturn: @@ -177,7 +171,7 @@ def dump_jshint_error() -> None: except ValueError: dump_jshint_error() - jshint_errors: List[str] = [] + jshint_errors: list[str] = [] js_text_lines = js_text.split("\n") @@ -193,7 +187,7 @@ def dump_jshint_error() -> None: return JSHintJSReturn(jshint_errors, jshint_json.get("globals", [])) -def print_js_hint_messages(js_hint_messages: List[str], source_line: Optional[SourceLine]) -> None: +def print_js_hint_messages(js_hint_messages: list[str], source_line: Optional[SourceLine]) -> None: """Log the message from JSHint, using the line number.""" if source_line is not None: for js_hint_message in js_hint_messages: @@ -203,7 +197,7 @@ def print_js_hint_messages(js_hint_messages: List[str], source_line: Optional[So def validate_js_expressions( tool: CommentedMap, schema: Schema, - jshint_options: Optional[Dict[str, Union[List[str], str, int]]] = None, + jshint_options: Optional[dict[str, Union[list[str], str, int]]] = None, container_engine: str = "docker", eval_timeout: float = 60, ) -> None: diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 982ec7e70..a6e2ba189 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -3,16 +3,8 @@ import functools import logging import random -from typing import ( - Callable, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - cast, -) +from collections.abc import Mapping, MutableMapping, MutableSequence +from typing import Callable, Optional, cast from uuid import UUID from mypy_extensions import mypyc_attr @@ -80,8 +72,7 @@ def __init__( if is_main: run_uuid = loadingContext.research_obj.ro_uuid - self.provenance_object = ProvenanceProfile( - loadingContext.research_obj, + self.provenance_object = loadingContext.research_obj.initialize_provenance( full_name=loadingContext.cwl_full_name, host_provenance=loadingContext.host_provenance, user_provenance=loadingContext.user_provenance, @@ -98,7 +89,7 @@ def __init__( loadingContext.requirements = self.requirements loadingContext.hints = self.hints - self.steps: List[WorkflowStep] = [] + self.steps: list[WorkflowStep] = [] validation_errors = [] for index, step in enumerate(self.tool.get("steps", [])): try: @@ -119,9 +110,9 @@ def __init__( workflow_inputs = self.tool["inputs"] workflow_outputs = self.tool["outputs"] - step_inputs: List[CWLObjectType] = [] - step_outputs: List[CWLObjectType] = [] - param_to_step: Dict[str, CWLObjectType] = {} + step_inputs: list[CWLObjectType] = [] + step_outputs: list[CWLObjectType] = [] + param_to_step: dict[str, CWLObjectType] = {} for step in self.steps: step_inputs.extend(step.tool["inputs"]) step_outputs.extend(step.tool["outputs"]) @@ -220,7 +211,7 @@ def __init__( loadingContext.requirements.append(parent_req) loadingContext.requirements.extend( cast( - List[CWLObjectType], + list[CWLObjectType], get_overrides(getdefault(loadingContext.overrides_list, []), self.id).get( "requirements", [] ), @@ -451,7 +442,9 @@ def job( runtimeContext, ) except WorkflowException: - _logger.error("Exception on step '%s'", runtimeContext.name) + _logger.error( + "Exception on step '%s'", runtimeContext.name, exc_info=runtimeContext.debug + ) raise except Exception as exc: _logger.exception("Unexpected exception") diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 2e69ca70c..b552641e1 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -3,18 +3,8 @@ import functools import logging import threading -from typing import ( - TYPE_CHECKING, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Sized, - Tuple, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence, Sized +from typing import TYPE_CHECKING, Optional, Union, cast from cwl_utils import expression from schema_salad.sourceline import SourceLine @@ -88,11 +78,16 @@ def __init__( ) -> None: """Initialize.""" self.dest = dest - self.completed = 0 + self._completed: set[int] = set() self.processStatus = "success" self.total = total self.output_callback = output_callback - self.steps: List[Optional[JobsGeneratorType]] = [] + self.steps: list[Optional[JobsGeneratorType]] = [] + + @property + def completed(self) -> int: + """The number of completed internal jobs.""" + return len(self._completed) def receive_scatter_output(self, index: int, jobout: CWLObjectType, processStatus: str) -> None: """Record the results of a scatter operation.""" @@ -108,15 +103,16 @@ def receive_scatter_output(self, index: int, jobout: CWLObjectType, processStatu if self.processStatus != "permanentFail": self.processStatus = processStatus - self.completed += 1 + if index not in self._completed: + self._completed.add(index) - if self.completed == self.total: - self.output_callback(self.dest, self.processStatus) + if self.completed == self.total: + self.output_callback(self.dest, self.processStatus) def setTotal( self, total: int, - steps: List[Optional[JobsGeneratorType]], + steps: list[Optional[JobsGeneratorType]], ) -> None: """ Set the total number of expected outputs along with the steps. @@ -130,7 +126,7 @@ def setTotal( def parallel_steps( - steps: List[Optional[JobsGeneratorType]], + steps: list[Optional[JobsGeneratorType]], rc: ReceiveScatterOutput, runtimeContext: RuntimeContext, ) -> JobsGeneratorType: @@ -180,7 +176,7 @@ def nested_crossproduct_scatter( rc = ReceiveScatterOutput(output_callback, output, jobl) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[Optional[JobsGeneratorType]] = [] for index in range(0, jobl): sjob: Optional[CWLObjectType] = copy.copy(joborder) assert sjob is not None # nosec @@ -247,11 +243,11 @@ def _flat_crossproduct_scatter( callback: ReceiveScatterOutput, startindex: int, runtimeContext: RuntimeContext, -) -> Tuple[List[Optional[JobsGeneratorType]], int]: +) -> tuple[list[Optional[JobsGeneratorType]], int]: """Inner loop.""" scatter_key = scatter_keys[0] jobl = len(cast(Sized, joborder[scatter_key])) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[Optional[JobsGeneratorType]] = [] put = startindex for index in range(0, jobl): sjob: Optional[CWLObjectType] = copy.copy(joborder) @@ -302,7 +298,7 @@ def dotproduct_scatter( rc = ReceiveScatterOutput(output_callback, output, jobl) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[Optional[JobsGeneratorType]] = [] for index in range(0, jobl): sjobo: Optional[CWLObjectType] = copy.copy(joborder) assert sjobo is not None # nosec @@ -350,7 +346,7 @@ def match_types( elif linkMerge: if iid not in inputobj: inputobj[iid] = [] - sourceTypes = cast(List[Optional[CWLOutputType]], inputobj[iid]) + sourceTypes = cast(list[Optional[CWLOutputType]], inputobj[iid]) if linkMerge == "merge_nested": sourceTypes.append(src.value) elif linkMerge == "merge_flattened": @@ -373,7 +369,7 @@ def match_types( def object_from_state( - state: Dict[str, Optional[WorkflowStateItem]], + state: dict[str, Optional[WorkflowStateItem]], params: ParametersType, frag_only: bool, supportsMultipleInput: bool, @@ -410,7 +406,7 @@ def object_from_state( ("merge_nested" if len(connections) > 1 else None), ), ), - valueFrom=cast(str, inp.get("valueFrom")), + valueFrom=cast(Optional[str], inp.get("valueFrom")), ): raise WorkflowException( "Type mismatch between source '%s' (%s) and " @@ -480,7 +476,7 @@ def __init__(self, workflow: "Workflow", runtimeContext: RuntimeContext) -> None self.prov_obj = workflow.provenance_object self.parent_wf = workflow.parent_wf self.steps = [WorkflowJobStep(s) for s in workflow.steps] - self.state: Dict[str, Optional[WorkflowStateItem]] = {} + self.state: dict[str, Optional[WorkflowStateItem]] = {} self.processStatus = "" self.did_callback = False self.made_progress: Optional[bool] = None @@ -547,7 +543,7 @@ def do_output_callback(self, final_output_callback: OutputCallbackType) -> None: def receive_output( self, step: WorkflowJobStep, - outputparms: List[CWLObjectType], + outputparms: list[CWLObjectType], final_output_callback: OutputCallbackType, jobout: CWLObjectType, processStatus: str, @@ -694,7 +690,7 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType] return psio if "scatter" in step.tool: - scatter = cast(List[str], aslist(step.tool["scatter"])) + scatter = cast(list[str], aslist(step.tool["scatter"])) method = step.tool.get("scatterMethod") if method is None and len(scatter) != 1: raise WorkflowException( @@ -954,7 +950,7 @@ def loop_callback( try: loop = cast(MutableSequence[CWLObjectType], self.step.tool.get("loop", [])) outputMethod = self.step.tool.get("outputMethod", "last_iteration") - state: Dict[str, Optional[WorkflowStateItem]] = {} + state: dict[str, Optional[WorkflowStateItem]] = {} for i in self.step.tool["outputs"]: if "id" in i: iid = cast(str, i["id"]) diff --git a/docs/cli.rst b/docs/cli.rst index d569f5586..1c6021bf4 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -3,4 +3,4 @@ cwltool Command Line Options .. autoprogram:: cwltool.argparser:arg_parser() :prog: cwltool - + :groups: diff --git a/docs/conf.py b/docs/conf.py index 6e04b5d64..e476f4191 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,8 @@ intersphinx_mapping = { "python": ("https://docs.python.org/3", None), "schema_salad": ("https://schema-salad.readthedocs.io/en/stable/", None), - "rdflib": ("https://rdflib.readthedocs.io/en/6.2.0/", None), + "rdflib": ("https://rdflib.readthedocs.io/en/stable/", None), + "cwl_utils": ("https://cwl-utils.readthedocs.io/en/stable/", None), # "ruamel.yaml": ("https://yaml.readthedocs.io/en/stable/", None), } diff --git a/docs/requirements.txt b/docs/requirements.txt index 40a2eefbf..fd3033b91 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ sphinx >= 2.2 -sphinx-rtd-theme==2.0.0 +sphinx-rtd-theme==3.0.2 sphinx-autoapi sphinx-autodoc-typehints sphinxcontrib-autoprogram diff --git a/lint-requirements.txt b/lint-requirements.txt index b0d7944eb..a2c6768cf 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ -flake8-bugbear<24.9 -black~=24.8 +flake8-bugbear<24.13 +black==25.* codespell diff --git a/mypy-requirements.txt b/mypy-requirements.txt index a7d0dacb8..65e517172 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,9 +1,10 @@ -mypy==1.11.2 # also update pyproject.toml +mypy==1.15.0 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 +cwltest types-requests types-setuptools types-psutil types-mock -galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2 -galaxy-util<24.2 +galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.3 +galaxy-util<24.3 diff --git a/mypy-stubs/black/__init__.pyi b/mypy-stubs/black/__init__.pyi deleted file mode 100644 index f741ef771..000000000 --- a/mypy-stubs/black/__init__.pyi +++ /dev/null @@ -1,26 +0,0 @@ -import asyncio -from concurrent.futures import Executor -from enum import Enum -from pathlib import Path -from typing import ( - Any, - Iterator, - List, - MutableMapping, - Optional, - Pattern, - Set, - Sized, - Tuple, - Union, -) - -from black.mode import Mode as Mode -from black.mode import TargetVersion as TargetVersion - -FileContent = str -Encoding = str -NewLine = str -FileMode = Mode - -def format_str(src_contents: str, mode: Mode) -> FileContent: ... diff --git a/mypy-stubs/mistune.pyi b/mypy-stubs/mistune.pyi deleted file mode 100644 index 3778c9195..000000000 --- a/mypy-stubs/mistune.pyi +++ /dev/null @@ -1,197 +0,0 @@ -__author__ = "Aleksandr Slepchenkov" -__email__ = "Sl.aleksandr28@gmail.com" - -from typing import ( - Any, - Dict, - Iterable, - List, - Match, - Optional, - Pattern, - Sequence, - Tuple, - Type, -) - -Tokens = List[Dict[str, Any]] -# There are too much levels of optional unions of lists of text in cell and align 385 and 396 lines in mistune - -def escape(text: str, quote: bool = ..., smart_amp: bool = ...) -> str: ... - -class BlockGrammar: - def_links: Pattern[str] - def_footnotes: Pattern[str] - newline: Pattern[str] - block_code: Pattern[str] - fences: Pattern[str] - hrule: Pattern[str] - heading: Pattern[str] - lheading: Pattern[str] - block_quote: Pattern[str] - list_block: Pattern[str] - list_item: Pattern[str] - list_bullet: Pattern[str] - paragraph: Pattern[str] - block_html: Pattern[str] - table: Pattern[str] - nptable: Pattern[str] - text: Pattern[str] - -class BlockLexer: - grammar_class: Type[BlockGrammar] - default_rules: List[str] - list_rules: Tuple[str] - footnote_rules: Tuple[str] - tokens: Tokens - def_links: Dict[str, Dict[str, str]] - def_footnotes: Dict[str, int] - rules = ... # type: BlockGrammar - def __init__(self, rules: Optional[BlockGrammar] = ..., **kwargs: Any) -> None: ... - def __call__(self, text: str, rules: Optional[Sequence[str]] = ...) -> Tokens: ... - def parse(self, text: str, rules: Optional[Sequence[str]] = ...) -> Tokens: ... - def parse_newline(self, m: Match[str]) -> None: ... - def parse_block_code(self, m: Match[str]) -> None: ... - def parse_fences(self, m: Match[str]) -> None: ... - def parse_heading(self, m: Match[str]) -> None: ... - def parse_lheading(self, m: Match[str]) -> None: ... - def parse_hrule(self, m: Match[str]) -> None: ... - def parse_list_block(self, m: Match[str]) -> None: ... - def parse_block_quote(self, m: Match[str]) -> None: ... - def parse_def_links(self, m: Match[str]) -> None: ... - def parse_def_footnotes(self, m: Match[str]) -> None: ... - def parse_table(self, m: Match[str]) -> None: ... - def parse_nptable(self, m: Match[str]) -> None: ... - def parse_block_html(self, m: Match[str]) -> None: ... - def parse_paragraph(self, m: Match[str]) -> None: ... - def parse_text(self, m: Match[str]) -> None: ... - -class InlineGrammar: - escape: Pattern[str] - inline_html: Pattern[str] - autolink: Pattern[str] - link: Pattern[str] - reflink: Pattern[str] - nolink: Pattern[str] - url: Pattern[str] - double_emphasis: Pattern[str] - emphasis: Pattern[str] - code: Pattern[str] - linebreak: Pattern[str] - strikethrough: Pattern[str] - footnote: Pattern[str] - text: Pattern[str] - def hard_wrap(self) -> None: ... - -class InlineLexer: - grammar_class: Type[InlineGrammar] - default_rules: List[str] - inline_html_rules: List[str] - renderer: Renderer - links: Dict[str, Dict[str, str]] - footnotes: Dict[str, int] - footnote_index: int - _in_link: bool - _in_footnote: bool - _parse_inline_html: bool - rules: InlineGrammar - def __init__( - self, renderer: Renderer, rules: Optional[InlineGrammar] = ..., **kwargs: Any - ) -> None: ... - def __call__(self, text: str, rules: Optional[Sequence[str]] = ...) -> str: ... - def setup( - self, - links: Optional[Dict[str, Dict[str, str]]], - footnotes: Optional[Dict[str, int]], - ) -> None: ... - line_match: Match[str] - line_started: bool - def output(self, text: str, rules: Optional[Sequence[str]] = ...) -> str: ... - def output_escape(self, m: Match[str]) -> str: ... - def output_autolink(self, m: Match[str]) -> str: ... - def output_url(self, m: Match[str]) -> str: ... - def output_inline_html(self, m: Match[str]) -> str: ... - def output_footnote(self, m: Match[str]) -> Optional[str]: ... - def output_link(self, m: Match[str]) -> str: ... - def output_reflink(self, m: Match[str]) -> Optional[str]: ... - def output_nolink(self, m: Match[str]) -> Optional[str]: ... - def output_double_emphasis(self, m: Match[str]) -> str: ... - def output_emphasis(self, m: Match[str]) -> str: ... - def output_code(self, m: Match[str]) -> str: ... - def output_linebreak(self, m: Match[str]) -> str: ... - def output_strikethrough(self, m: Match[str]) -> str: ... - def output_text(self, m: Match[str]) -> str: ... - -class Renderer: - options: Dict[str, str] - def __init__(self, **kwargs: Any) -> None: ... - def placeholder(self) -> str: ... - def block_code( - self, code: str, lang: Any = ... - ) -> str: ... # It seems that lang should be string, however other types are valid as well - def block_quote(self, text: str) -> str: ... - def block_html(self, html: str) -> str: ... - def header(self, text: str, level: int, raw: Optional[str] = ...) -> str: ... - def hrule(self) -> str: ... - def list( - self, body: Any, ordered: bool = ... - ) -> str: ... # body - same reason as for lang above, and for other Any in this class - def list_item(self, text: Any) -> str: ... - def paragraph(self, text: str) -> str: ... - def table(self, header: Any, body: Any) -> str: ... - def table_row(self, content: Any) -> str: ... - def table_cell(self, content: Any, **flags: Dict[str, Any]) -> str: ... - def double_emphasis(self, text: Any) -> str: ... - def emphasis(self, text: Any) -> str: ... - def codespan(self, text: str) -> str: ... - def linebreak(self) -> str: ... - def strikethrough(self, text: Any) -> str: ... - def text(self, text: Any) -> str: ... - def escape(self, text: Any) -> str: ... - def autolink(self, link: Any, is_email: bool = ...) -> str: ... - def link(self, link: Any, title: Any, text: Any) -> str: ... - def image(self, src: Any, title: Any, text: Any) -> str: ... - def inline_html(self, html: Any) -> str: ... - def newline(self) -> str: ... - def footnote_ref(self, key: Any, index: int) -> str: ... - def footnote_item(self, key: Any, text: str) -> str: ... - def footnotes(self, text: Any) -> str: ... - -class Markdown: - renderer = ... # type: Renderer - inline = ... # type: InlineLexer - block = ... # type: BlockLexer - footnotes = ... # type: List[Dict[str, Any]] - tokens = ... # type: Tokens - def __init__( - self, - renderer: Optional[Renderer] = ..., - inline: Optional[InlineLexer] = ..., - block: Optional[BlockLexer] = ..., - **kwargs: Any, - ) -> None: ... - def __call__(self, text: str) -> str: ... - def render(self, text: str) -> str: ... - def parse(self, text: str) -> str: ... - token = ... # type: Dict[str, Any] - def pop(self) -> Optional[Dict[str, Any]]: ... - def peek(self) -> Optional[Dict[str, Any]]: ... - def output(self, text: str, rules: Optional[Sequence[str]] = ...) -> str: ... - def tok(self) -> str: ... - def tok_text(self) -> str: ... - def output_newline(self) -> str: ... - def output_hrule(self) -> str: ... - def output_heading(self) -> str: ... - def output_code(self) -> str: ... - def output_table(self) -> str: ... - def output_block_quote(self) -> str: ... - def output_list(self) -> str: ... - def output_list_item(self) -> str: ... - def output_loose_item(self) -> str: ... - def output_footnote(self) -> str: ... - def output_close_html(self) -> str: ... - def output_open_html(self) -> str: ... - def output_paragraph(self) -> str: ... - def output_text(self) -> str: ... - -def markdown(text: str, escape: bool = ..., **kwargs: Any) -> str: ... diff --git a/mypy-stubs/rdflib/graph.pyi b/mypy-stubs/rdflib/graph.pyi index d3e6f2f54..9764972b2 100644 --- a/mypy-stubs/rdflib/graph.pyi +++ b/mypy-stubs/rdflib/graph.pyi @@ -16,7 +16,7 @@ from rdflib import query from rdflib.collection import Collection from rdflib.paths import Path from rdflib.resource import Resource -from rdflib.term import BNode, Identifier, Node +from rdflib.term import BNode, Identifier, Literal, Node class Graph(Node): base: Any = ... @@ -66,7 +66,7 @@ class Graph(Node): ) -> Iterable[Node]: ... def objects( self, subject: Optional[Any] = ..., predicate: Optional[Any] = ... - ) -> Iterable[Identifier]: ... + ) -> Iterable[Union[Identifier, Literal]]: ... def subject_predicates(self, object: Optional[Any] = ...) -> None: ... def subject_objects(self, predicate: Optional[Any] = ...) -> None: ... def predicate_objects(self, subject: Optional[Any] = ...) -> None: ... diff --git a/mypy-stubs/shellescape/__init__.pyi b/mypy-stubs/shellescape/__init__.pyi deleted file mode 100644 index 621241e5e..000000000 --- a/mypy-stubs/shellescape/__init__.pyi +++ /dev/null @@ -1,5 +0,0 @@ -# Stubs for shellescape (Python 2) -# -# NOTE: This dynamically typed stub was automatically generated by stubgen. - -from .main import quote as quote diff --git a/mypy-stubs/shellescape/main.pyi b/mypy-stubs/shellescape/main.pyi deleted file mode 100644 index 69eade63e..000000000 --- a/mypy-stubs/shellescape/main.pyi +++ /dev/null @@ -1,5 +0,0 @@ -# Stubs for shellescape.main (Python 2) - -from typing import AnyStr - -def quote(s: AnyStr) -> AnyStr: ... diff --git a/mypy.ini b/mypy.ini index bac992869..02545dce5 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,6 +5,7 @@ show_column_numbers = true show_error_codes = true pretty = true warn_unreachable = True +local_partial_types = true [mypy-galaxy.tool_util.*] ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml index 05a2b82f7..cb7d837a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,21 +2,35 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.11.2", # also update mypy-requirements.txt + "mypy==1.15.0", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", - "ruamel.yaml>=0.16.0,<0.18", + "ruamel.yaml>=0.16.0,<0.19", "schema-salad>=8.7,<9", "cwl-utils>=0.32", "toml", "argcomplete>=1.12.0", + "rich-argparse" ] build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "cwltool/_version.py" +[tool.cibuildwheel] +test-command = "python -m pytest --ignore cwltool/schemas -n logical --dist worksteal --junitxml={project}/test-results/junit_$(python -V | awk '{print $2}')_${AUDITWHEEL_PLAT}.xml -k 'not (test_bioconda or test_env_filtering or test_udocker)' --pyargs cwltool" +test-requires = "-r test-requirements.txt" +test-extras = "deps" +skip = "pp*" +# ^ skip building wheels on PyPy (any version) +build-verbosity = 1 +environment = { CWLTOOL_USE_MYPYC="1", MYPYPATH="$(pwd)/mypy-stubs" } + +# Install system library +[tool.cibuildwheel.linux] +before-all = "apk add libxml2-dev libxslt-dev nodejs || yum install -y libxml2-devel libxslt-devel nodejs environment-modules || apt-get install -y --no-install-recommends libxml2-dev libxslt-dev nodejs environment-modules" + [tool.black] line-length = 100 -target-version = [ "py38" ] +target-version = [ "py39" ] diff --git a/requirements.txt b/requirements.txt index df82a6f43..3cbcf0027 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ requests>=2.6.1 ruamel.yaml>=0.16.0,<0.19 -rdflib>=4.2.2,<7.1 -shellescape>=3.4.1,<3.9 +rdflib>=4.2.2,<7.2 schema-salad>=8.7,<9 prov==1.5.1 mypy-extensions @@ -13,3 +12,4 @@ argcomplete>=1.12.0 pyparsing!=3.0.2 # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 cwl-utils>=0.32 spython>=0.3.0 +rich-argparse diff --git a/setup.py b/setup.py index 9980276e5..4aed7320b 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,15 @@ #!/usr/bin/env python3 """Setup for the reference implementation of the CWL standards.""" +import glob import os import sys import warnings +from typing import TYPE_CHECKING, Any -from setuptools import setup +from setuptools import Extension, setup + +if TYPE_CHECKING: + from typing_extensions import TypeGuard if os.name == "nt": warnings.warn( @@ -20,6 +25,31 @@ stacklevel=1, ) + +def _is_list_of_setuptools_extension(items: list[Any]) -> "TypeGuard[list[Extension]]": + return all(isinstance(item, Extension) for item in items) + + +def _find_package_data(base: str, globs: list[str], root: str = "cwltool") -> list[str]: + """ + Find all interesting data files, for setup(package_data=). + + Arguments: + root: The directory to search in. + globs: A list of glob patterns to accept files. + """ + rv_dirs = [root for root, dirs, files in os.walk(base)] + rv = [] + for rv_dir in rv_dirs: + files = [] + for pat in globs: + files += glob.glob(os.path.join(rv_dir, pat)) + if not files: + continue + rv.extend([os.path.relpath(f, root) for f in files]) + return rv + + SETUP_DIR = os.path.dirname(__file__) README = os.path.join(SETUP_DIR, "README.rst") @@ -34,55 +64,50 @@ USE_MYPYC = True if USE_MYPYC: - mypyc_targets = [ - "cwltool/argparser.py", - "cwltool/builder.py", - "cwltool/checker.py", - "cwltool/command_line_tool.py", - # "cwltool/context.py", # monkeypatching - "cwltool/cwlrdf.py", - "cwltool/docker_id.py", - "cwltool/docker.py", - "cwltool/udocker.py", - "cwltool/errors.py", - "cwltool/executors.py", - "cwltool/factory.py", - "cwltool/flatten.py", - # "cwltool/__init__.py", - "cwltool/job.py", - "cwltool/load_tool.py", - # "cwltool/loghandler.py", # so we can monkeypatch the logger from tests - # "cwltool/__main__.py", - "cwltool/main.py", - "cwltool/mutation.py", - "cwltool/pack.py", - "cwltool/pathmapper.py", - "cwltool/process.py", - "cwltool/procgenerator.py", - # "cwltool/cwlprov/__init__.py", - "cwltool/cwlprov/provenance_constants.py", - "cwltool/cwlprov/provenance_profile.py", - "cwltool/cwlprov/ro.py", - # "cwltool/cwlprov/writablebagfile.py", # WritableBag is having issues - "cwltool/resolver.py", - "cwltool/secrets.py", - "cwltool/singularity.py", - "cwltool/software_requirements.py", - # "cwltool/stdfsaccess.py", # StdFsAccess needs to be subclassable - "cwltool/subgraph.py", - "cwltool/update.py", - "cwltool/utils.py", - "cwltool/validate_js.py", - "cwltool/workflow.py", + mypyc_skiplist = tuple( + os.path.join("cwltool", x) + for x in ( + "context.py", # monkeypatching + "__init__.py", + "loghandler.py", # so we can monkeypatch the logger from tests + "__main__.py", + "cwlprov/__init__.py", + "cuda.py", # for monkeypatch + "run_job.py", + "cwlprov/writablebagfile.py", # WritableBag is having issues + "stdfsaccess.py", # StdFsAccess needs to be subclassable + ) + ) + + everything = [os.path.join("cwltool", x) for x in _find_package_data("cwltool", ["*.py"])] + # Start with all the .py files + all_real_pys = [ + x for x in everything if not x.startswith(os.path.join("mypy", "typeshed") + os.sep) ] + # Strip out anything in our skiplist + mypyc_targets = [x for x in all_real_pys if x not in mypyc_skiplist] + + # Strip out any test code + mypyc_targets = [x for x in mypyc_targets if not x.startswith(("tests" + os.sep))] - from mypyc.build import mypycify # type: ignore[import-untyped] + mypyc_targets.sort() + + from mypyc.build import mypycify opt_level = os.getenv("MYPYC_OPT_LEVEL", "3") - ext_modules = mypycify(mypyc_targets, opt_level=opt_level) + debug_level = os.getenv("MYPYC_DEBUG_LEVEL", "1") + force_multifile = os.getenv("MYPYC_MULTI_FILE", "") == "1" + ext_modules = mypycify( + mypyc_targets, + opt_level=opt_level, + debug_level=debug_level, + multi_file=force_multifile, + ) else: ext_modules = [] +assert _is_list_of_setuptools_extension(ext_modules), "Expected mypycify to use setuptools" + setup( name="cwltool", description="Common workflow language reference implementation", @@ -121,31 +146,29 @@ package_dir={"cwltool.tests": "tests"}, include_package_data=True, install_requires=[ - "setuptools", "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 "ruamel.yaml >= 0.16, < 0.19", - "rdflib >= 4.2.2, < 7.1.0", - "shellescape >= 3.4.1, < 3.9", + "rdflib >= 4.2.2, < 7.2.0", "schema-salad >= 8.7, < 9", "prov == 1.5.1", "mypy-extensions", "psutil >= 5.6.6", - "importlib_resources>=1.4;python_version<'3.9'", "coloredlogs", "pydot >= 1.4.1, <3", - "argcomplete", + "argcomplete >= 1.12.0", "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 "cwl-utils >= 0.32", "spython >= 0.3.0", + "rich-argparse", ], extras_require={ "deps": [ - "galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2", - "galaxy-util <24.2", + "galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.3", + "galaxy-util <24.3", ], }, - python_requires=">=3.8, <3.14", + python_requires=">=3.9, <3.14", use_scm_version=True, setup_requires=PYTEST_RUNNER + ["setuptools_scm>=8.0.4,<9"], test_suite="tests", @@ -171,7 +194,6 @@ "Operating System :: POSIX", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/test-requirements.txt b/test-requirements.txt index e545ee65a..c5ca6ef73 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,11 +3,11 @@ pytest>= 6.2,< 8.4 pytest-xdist>=3.2.0 # for the worksteal scheduler psutil # enhances pytest-xdist to allow "-n logical" pytest-httpserver -pytest-retry;python_version>'3.9' +pytest-retry;python_version>='3.9' mock>=2.0.0 pytest-mock>=1.10.0 pytest-cov arcp>=0.2.0 -r requirements.txt -galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2 -galaxy-util<24.2 +galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.3 +galaxy-util<24.3 diff --git a/tests/CometAdapter.cwl b/tests/CometAdapter.cwl new file mode 100755 index 000000000..3d7083560 --- /dev/null +++ b/tests/CometAdapter.cwl @@ -0,0 +1,229 @@ +#!/usr/bin/env cwl-runner +# Copyright (c) 2002-present, The OpenMS Team -- EKU Tuebingen, ETH Zurich, and FU Berlin +# SPDX-License-Identifier: Apache-2.0 +label: CometAdapter +doc: Annotates MS/MS spectra using Comet. +inputs: + in: + doc: Input file + type: File + out: + doc: Output file + type: string + database: + doc: FASTA file + type: File + comet_executable: + doc: The Comet executable. Provide a full or relative path, or make sure it can be found in your PATH environment. + type: File + pin_out: + doc: Output file - for Percolator input + type: string? + default_params_file: + doc: Default Comet params file. All parameters of this take precedence. A template file can be generated using 'comet.exe -p' + type: File? + precursor_mass_tolerance: + doc: "Precursor monoisotopic mass tolerance (Comet parameter: peptide_mass_tolerance). See also precursor_error_units to set the unit." + type: double? + precursor_error_units: + doc: "Unit of precursor monoisotopic mass tolerance for parameter precursor_mass_tolerance (Comet parameter: peptide_mass_units)" + type: string? + isotope_error: + doc: This parameter controls whether the peptide_mass_tolerance takes into account possible isotope errors in the precursor mass measurement. Use -8/-4/0/4/8 only for SILAC. + type: string? + fragment_mass_tolerance: + doc: "This is half the bin size, which is used to segment the MS/MS spectrum. Thus, the value should be a bit higher than for other search engines, since the bin might not be centered around the peak apex (see 'fragment_bin_offset').CAUTION: Low tolerances have heavy impact on RAM usage (since Comet uses a lot of bins in this case). Consider using use_sparse_matrix and/or spectrum_batch_size." + type: double? + fragment_error_units: + doc: Fragment monoisotopic mass error units + type: string? + fragment_bin_offset: + doc: "Offset of fragment bins. Recommended by Comet: low-res: 0.4, high-res: 0.0" + type: double? + instrument: + doc: "Comets theoretical_fragment_ions parameter: theoretical fragment ion peak representation, high-res: sum of intensities plus flanking bins, ion trap (low-res) ms/ms: sum of intensities of central M bin only" + type: string? + use_A_ions: + doc: use A ions for PSM + type: boolean? + use_B_ions: + doc: use B ions for PSM + type: boolean? + use_C_ions: + doc: use C ions for PSM + type: boolean? + use_X_ions: + doc: use X ions for PSM + type: boolean? + use_Y_ions: + doc: use Y ions for PSM + type: boolean? + use_Z_ions: + doc: use Z ions for PSM + type: boolean? + use_NL_ions: + doc: use neutral loss (NH3, H2O) ions from b/y for PSM + type: boolean? + enzyme: + doc: The enzyme used for peptide digestion. + type: string? + second_enzyme: + doc: Additional enzyme used for peptide digestion. + type: string? + num_enzyme_termini: + doc: Specify the termini where the cleavage rule has to match + type: string? + missed_cleavages: + doc: Number of possible cleavage sites missed by the enzyme. It has no effect if enzyme is unspecific cleavage. + type: long? + min_peptide_length: + doc: Minimum peptide length to consider. + type: long? + max_peptide_length: + doc: Maximum peptide length to consider. + type: long? + num_hits: + doc: Number of peptide hits (PSMs) per spectrum in output file + type: long? + precursor_charge: + doc: "Precursor charge range to search (if spectrum is not annotated with a charge or if override_charge!=keep any known): 0:[num] == search all charges, 2:6 == from +2 to +6, 3:3 == +3" + type: string? + override_charge: + doc: "_keep any known_: keep any precursor charge state (from input), _ignore known_: ignore known precursor charge state and use precursor_charge parameter, _ignore outside range_: ignore precursor charges outside precursor_charge range, _keep known search unknown_: keep any known precursor charge state. For unknown charge states, search as singly charged if there is no signal above the precursor m/z or use the precursor_charge range" + type: string? + ms_level: + doc: MS level to analyze, valid are levels 2 (default) or 3 + type: long? + activation_method: + doc: If not ALL, only searches spectra of the given method + type: string? + digest_mass_range: + doc: MH+ peptide mass range to analyze + type: string? + max_fragment_charge: + doc: Set maximum fragment charge state to analyze as long as still lower than precursor charge - 1. (Allowed max 5) + type: long? + max_precursor_charge: + doc: set maximum precursor charge state to analyze (allowed max 9) + type: long? + clip_nterm_methionine: + doc: If set to true, also considers the peptide sequence w/o N-term methionine separately and applies appropriate N-term mods to it + type: boolean? + spectrum_batch_size: + doc: max. number of spectra to search at a time; use 0 to search the entire scan range in one batch + type: long? + mass_offsets: + doc: One or more mass offsets to search (values subtracted from deconvoluted precursor mass). Has to include 0.0 if you want the default mass to be searched. + type: double[]? + minimum_peaks: + doc: Required minimum number of peaks in spectrum to search (default 10) + type: long? + minimum_intensity: + doc: Minimum intensity value to read in + type: double? + remove_precursor_peak: + doc: no = no removal, yes = remove all peaks around precursor m/z, charge_reduced = remove all charge reduced precursor peaks (for ETD/ECD). phosphate_loss = remove the HPO3 (-80) and H3PO4 (-98) precursor phosphate neutral loss peaks. See also remove_precursor_tolerance + type: string? + remove_precursor_tolerance: + doc: one-sided tolerance for precursor removal in Thompson + type: double? + clear_mz_range: + doc: for iTRAQ/TMT type data; will clear out all peaks in the specified m/z range, if not 0:0 + type: string? + fixed_modifications: + doc: Fixed modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + type: string[]? + variable_modifications: + doc: Variable modifications, specified using Unimod (www.unimod.org) terms, e.g. 'Carbamidomethyl (C)' or 'Oxidation (M)' + type: string[]? + binary_modifications: + doc: "List of modification group indices. Indices correspond to the binary modification index used by comet to group individually searched lists of variable modifications.\nNote: if set, both variable_modifications and binary_modifications need to have the same number of entries as the N-th entry corresponds to the N-th variable_modification.\n if left empty (default), all entries are internally set to 0 generating all permutations of modified and unmodified residues.\n For a detailed explanation please see the parameter description in the Comet help." + type: long[]? + max_variable_mods_in_peptide: + doc: Set a maximum number of variable modifications per peptide + type: long? + require_variable_mod: + doc: If true, requires at least one variable modification per peptide + type: boolean? + reindex: + doc: Recalculate peptide to protein association using OpenMS. Annotates target-decoy information. + type: string? + log: + doc: Name of log file (created only when specified) + type: string? + debug: + doc: Sets the debug level + type: long? + threads: + doc: Sets the number of threads allowed to be used by the TOPP tool + type: long? + no_progress: + doc: Disables progress logging to command line + type: boolean? + force: + doc: Overrides tool-specific checks + type: boolean? + test: + doc: Enables the test mode (needed for internal use only) + type: boolean? + PeptideIndexing__decoy_string: + doc: String that was appended (or prefixed - see 'decoy_string_position' flag below) to the accessions in the protein database to indicate decoy proteins. If empty (default), it's determined automatically (checking for common terms, both as prefix and suffix). + type: string? + PeptideIndexing__decoy_string_position: + doc: Is the 'decoy_string' prepended (prefix) or appended (suffix) to the protein accession? (ignored if decoy_string is empty) + type: string? + PeptideIndexing__missing_decoy_action: + doc: "Action to take if NO peptide was assigned to a decoy protein (which indicates wrong database or decoy string): 'error' (exit with error, no output), 'warn' (exit with success, warning message), 'silent' (no action is taken, not even a warning)" + type: string? + PeptideIndexing__write_protein_sequence: + doc: If set, the protein sequences are stored as well. + type: boolean? + PeptideIndexing__write_protein_description: + doc: If set, the protein description is stored as well. + type: boolean? + PeptideIndexing__keep_unreferenced_proteins: + doc: If set, protein hits which are not referenced by any peptide are kept. + type: boolean? + PeptideIndexing__unmatched_action: + doc: "If peptide sequences cannot be matched to any protein: 1) raise an error; 2) warn (unmatched PepHits will miss target/decoy annotation with downstream problems); 3) remove the hit." + type: string? + PeptideIndexing__aaa_max: + doc: Maximal number of ambiguous amino acids (AAAs) allowed when matching to a protein database with AAAs. AAAs are 'B', 'J', 'Z' and 'X'. + type: long? + PeptideIndexing__mismatches_max: + doc: Maximal number of mismatched (mm) amino acids allowed when matching to a protein database. The required runtime is exponential in the number of mm's; apply with care. MM's are allowed in addition to AAA's. + type: long? + PeptideIndexing__IL_equivalent: + doc: Treat the isobaric amino acids isoleucine ('I') and leucine ('L') as equivalent (indistinguishable). Also occurrences of 'J' will be treated as 'I' thus avoiding ambiguous matching. + type: boolean? + PeptideIndexing__allow_nterm_protein_cleavage: + doc: Allow the protein N-terminus amino acid to clip. + type: string? + PeptideIndexing__enzyme__name: + doc: "Enzyme which determines valid cleavage sites - e.g. trypsin cleaves after lysine (K) or arginine (R), but not before proline (P). Default: deduce from input" + type: string? + PeptideIndexing__enzyme__specificity: + doc: "Specificity of the enzyme. Default: deduce from input.\n 'full': both internal cleavage sites must match.\n 'semi': one of two internal cleavage sites must match.\n 'none': allow all peptide hits no matter their context (enzyme is irrelevant)." + type: string? +outputs: + out: + type: File + outputBinding: + glob: $(inputs.out) + pin_out: + type: File? + outputBinding: + glob: $(inputs.pin_out) +cwlVersion: v1.2 +class: CommandLineTool +baseCommand: + - CometAdapter +requirements: + InlineJavascriptRequirement: {} + InitialWorkDirRequirement: + listing: + - entryname: cwl_inputs.json + entry: $(JSON.stringify(inputs)) +arguments: + - -ini + - cwl_inputs.json diff --git a/tests/arg-empty-prefix-separate-false.cwl b/tests/arg-empty-prefix-separate-false.cwl old mode 100644 new mode 100755 diff --git a/tests/checker_wf/optional_array_mismatch.cwl b/tests/checker_wf/optional_array_mismatch.cwl old mode 100644 new mode 100755 index d6f052757..a557927f2 --- a/tests/checker_wf/optional_array_mismatch.cwl +++ b/tests/checker_wf/optional_array_mismatch.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: Workflow inputs: diff --git a/tests/cwl-conformance/cwltool-conftest.py b/tests/cwl-conformance/cwltool-conftest.py index 3e2b83990..c87cf0ef7 100644 --- a/tests/cwl-conformance/cwltool-conftest.py +++ b/tests/cwl-conformance/cwltool-conftest.py @@ -6,20 +6,20 @@ import json from io import StringIO -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Optional from cwltest import utils def pytest_cwl_execute_test( config: utils.CWLTestConfig, processfile: str, jobfile: Optional[str] -) -> Tuple[int, Optional[Dict[str, Any]]]: +) -> tuple[int, Optional[dict[str, Any]]]: """Use the CWL reference runner (cwltool) to execute tests.""" from cwltool import main from cwltool.errors import WorkflowException stdout = StringIO() - argsl: List[str] = [f"--outdir={config.outdir}"] + argsl: list[str] = [f"--outdir={config.outdir}"] if config.runner_quiet: argsl.append("--quiet") elif config.verbose: diff --git a/tests/default_values_list.cwl b/tests/default_values_list.cwl old mode 100644 new mode 100755 diff --git a/tests/echo-badposition-expr.cwl b/tests/echo-badposition-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/echo-position-expr.cwl b/tests/echo-position-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/echo-stdout-log-dir.cwl b/tests/echo-stdout-log-dir.cwl old mode 100644 new mode 100755 diff --git a/tests/env.cwl b/tests/env.cwl old mode 100644 new mode 100755 diff --git a/tests/env2.cwl b/tests/env2.cwl old mode 100644 new mode 100755 diff --git a/tests/env3.cwl b/tests/env3.cwl old mode 100644 new mode 100755 diff --git a/tests/env4.cwl b/tests/env4.cwl old mode 100644 new mode 100755 diff --git a/tests/input_deps/docker-array-secondaryfiles.cwl b/tests/input_deps/docker-array-secondaryfiles.cwl old mode 100644 new mode 100755 diff --git a/tests/iwdr_bad_expr.cwl b/tests/iwdr_bad_expr.cwl old mode 100644 new mode 100755 diff --git a/tests/iwdr_dir_literal_real_file.cwl b/tests/iwdr_dir_literal_real_file.cwl old mode 100644 new mode 100755 diff --git a/tests/load_contents-array.cwl b/tests/load_contents-array.cwl old mode 100644 new mode 100755 index f6b786ec6..eaad62436 --- a/tests/load_contents-array.cwl +++ b/tests/load_contents-array.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: "v1.2" class: CommandLineTool baseCommand: echo diff --git a/tests/loop-ext/all-output-loop-no-iteration.cwl b/tests/loop-ext/all-output-loop-no-iteration.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/all-output-loop.cwl b/tests/loop-ext/all-output-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/default-value-loop.cwl b/tests/loop-ext/default-value-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-command-line-tool.cwl b/tests/loop-ext/invalid-loop-command-line-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-expression-tool.cwl b/tests/loop-ext/invalid-loop-expression-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-hint.cwl b/tests/loop-ext/invalid-loop-hint.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-scatter.cwl b/tests/loop-ext/invalid-loop-scatter.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-when-exception.cwl b/tests/loop-ext/invalid-loop-when-exception.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-when.cwl b/tests/loop-ext/invalid-loop-when.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-loop-workflow.cwl b/tests/loop-ext/invalid-loop-workflow.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-multi-source-loop-no-requirement.cwl b/tests/loop-ext/invalid-multi-source-loop-no-requirement.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-no-loopWhen.cwl b/tests/loop-ext/invalid-no-loopWhen.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-non-boolean-loopWhen.cwl b/tests/loop-ext/invalid-non-boolean-loopWhen.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-non-boolean-loopWhen2.cwl b/tests/loop-ext/invalid-non-boolean-loopWhen2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/invalid-value-from-loop-no-requirement.cwl b/tests/loop-ext/invalid-value-from-loop-no-requirement.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/loop-inside-loop-all.cwl b/tests/loop-ext/loop-inside-loop-all.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/loop-inside-loop.cwl b/tests/loop-ext/loop-inside-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/loop-inside-scatter.cwl b/tests/loop-ext/loop-inside-scatter.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/multi-source-loop.cwl b/tests/loop-ext/multi-source-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/opt-var-loop.cwl b/tests/loop-ext/opt-var-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/scatter-inside-loop.cwl b/tests/loop-ext/scatter-inside-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/single-var-loop-no-iteration.cwl b/tests/loop-ext/single-var-loop-no-iteration.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/single-var-loop.cwl b/tests/loop-ext/single-var-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/two-vars-loop-2.cwl b/tests/loop-ext/two-vars-loop-2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/two-vars-loop.cwl b/tests/loop-ext/two-vars-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop-ext/value-from-loop.cwl b/tests/loop-ext/value-from-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/all-output-loop-no-iteration.cwl b/tests/loop/all-output-loop-no-iteration.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/all-output-loop.cwl b/tests/loop/all-output-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/default-value-loop.cwl b/tests/loop/default-value-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-loop-scatter.cwl b/tests/loop/invalid-loop-scatter.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-loop-when-exception.cwl b/tests/loop/invalid-loop-when-exception.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-loop-when-exception2.cwl b/tests/loop/invalid-loop-when-exception2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-multi-source-loop-no-requirement.cwl b/tests/loop/invalid-multi-source-loop-no-requirement.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-no-loopWhen.cwl b/tests/loop/invalid-no-loopWhen.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-non-boolean-loopWhen.cwl b/tests/loop/invalid-non-boolean-loopWhen.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-non-boolean-loopWhen2.cwl b/tests/loop/invalid-non-boolean-loopWhen2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/invalid-value-from-loop-no-requirement.cwl b/tests/loop/invalid-value-from-loop-no-requirement.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/loop-inside-loop-all.cwl b/tests/loop/loop-inside-loop-all.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/loop-inside-loop.cwl b/tests/loop/loop-inside-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/loop-inside-scatter.cwl b/tests/loop/loop-inside-scatter.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/multi-source-loop.cwl b/tests/loop/multi-source-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/opt-var-loop.cwl b/tests/loop/opt-var-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/scatter-inside-loop.cwl b/tests/loop/scatter-inside-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/single-var-loop-no-iteration.cwl b/tests/loop/single-var-loop-no-iteration.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/single-var-loop.cwl b/tests/loop/single-var-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/two-vars-loop-2.cwl b/tests/loop/two-vars-loop-2.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/two-vars-loop.cwl b/tests/loop/two-vars-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/loop/value-from-loop.cwl b/tests/loop/value-from-loop.cwl old mode 100644 new mode 100755 diff --git a/tests/nested-array.cwl b/tests/nested-array.cwl old mode 100644 new mode 100755 index 8272614fc..6aa3650b8 --- a/tests/nested-array.cwl +++ b/tests/nested-array.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool baseCommand: echo diff --git a/tests/non_portable.cwl b/tests/non_portable.cwl old mode 100644 new mode 100755 diff --git a/tests/non_portable2.cwl b/tests/non_portable2.cwl old mode 100644 new mode 100755 diff --git a/tests/override/env-tool.cwl b/tests/override/env-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/override/env-tool_v1.1.0-dev1.cwl b/tests/override/env-tool_v1.1.0-dev1.cwl old mode 100644 new mode 100755 diff --git a/tests/override/env-tool_v1.1.cwl b/tests/override/env-tool_v1.1.cwl old mode 100644 new mode 100755 diff --git a/tests/portable.cwl b/tests/portable.cwl old mode 100644 new mode 100755 diff --git a/tests/reloc/test.cwl b/tests/reloc/test.cwl old mode 100644 new mode 100755 index 41d37474e..c88cf48da --- a/tests/reloc/test.cwl +++ b/tests/reloc/test.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool inputs: diff --git a/tests/scatter_numbers.cwl b/tests/scatter_numbers.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files-bad.cwl b/tests/secondary-files-bad.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files-required-container.cwl b/tests/secondary-files-required-container.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files-required-missing-container.cwl b/tests/secondary-files-required-missing-container.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files-string-v1.cwl b/tests/secondary-files-string-v1.cwl old mode 100644 new mode 100755 diff --git a/tests/secondary-files.cwl b/tests/secondary-files.cwl old mode 100644 new mode 100755 diff --git a/tests/sing_pullfolder_test.cwl b/tests/sing_pullfolder_test.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/1432.cwl b/tests/subgraph/1432.cwl old mode 100644 new mode 100755 index 54d553875..41283abfe --- a/tests/subgraph/1432.cwl +++ b/tests/subgraph/1432.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: Workflow cwlVersion: v1.2 inputs: diff --git a/tests/subgraph/env-tool2.cwl b/tests/subgraph/env-tool2.cwl old mode 100644 new mode 100755 index f568e5450..14c8a6826 --- a/tests/subgraph/env-tool2.cwl +++ b/tests/subgraph/env-tool2.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: CommandLineTool cwlVersion: v1.2 inputs: diff --git a/tests/subgraph/env-tool2_no_env.cwl b/tests/subgraph/env-tool2_no_env.cwl old mode 100644 new mode 100755 index c4a6f6327..2b3711b77 --- a/tests/subgraph/env-tool2_no_env.cwl +++ b/tests/subgraph/env-tool2_no_env.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: CommandLineTool cwlVersion: v1.2 inputs: diff --git a/tests/subgraph/env-tool2_req.cwl b/tests/subgraph/env-tool2_req.cwl old mode 100644 new mode 100755 index 96b0bc3d1..943108056 --- a/tests/subgraph/env-tool2_req.cwl +++ b/tests/subgraph/env-tool2_req.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: CommandLineTool cwlVersion: v1.2 inputs: diff --git a/tests/subgraph/env-wf2.cwl b/tests/subgraph/env-wf2.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_hint_collision.cwl b/tests/subgraph/env-wf2_hint_collision.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_hint_req_collision.cwl b/tests/subgraph/env-wf2_hint_req_collision.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_long.cwl b/tests/subgraph/env-wf2_long.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_only_hint.cwl b/tests/subgraph/env-wf2_only_hint.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_req_collision.cwl b/tests/subgraph/env-wf2_req_collision.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_subwf-packed.cwl b/tests/subgraph/env-wf2_subwf-packed.cwl old mode 100644 new mode 100755 index ed119f0f2..e1c09649d --- a/tests/subgraph/env-wf2_subwf-packed.cwl +++ b/tests/subgraph/env-wf2_subwf-packed.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner { "$graph": [ { @@ -127,4 +128,4 @@ } ], "cwlVersion": "v1.2" -} \ No newline at end of file +} diff --git a/tests/subgraph/env-wf2_subwf.cwl b/tests/subgraph/env-wf2_subwf.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/env-wf2_subwf_b.cwl b/tests/subgraph/env-wf2_subwf_b.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/steplevel-resreq.cwl b/tests/subgraph/steplevel-resreq.cwl old mode 100644 new mode 100755 diff --git a/tests/subgraph/timelimit2-wf.cwl b/tests/subgraph/timelimit2-wf.cwl old mode 100644 new mode 100755 diff --git a/tests/test_caching.py b/tests/test_caching.py new file mode 100644 index 000000000..db3bf0246 --- /dev/null +++ b/tests/test_caching.py @@ -0,0 +1,172 @@ +import re +from pathlib import Path + +import pytest + +from .util import get_data, get_main_output, needs_docker + +test_factors = [(""), ("--parallel"), ("--debug"), ("--parallel --debug")] + + +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_wf_without_container(tmp_path: Path, factor: str) -> None: + """Confirm that we can run a workflow without a container.""" + test_file = "hello-workflow.cwl" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend( + [ + "--cachedir", + cache_dir, + "--outdir", + str(tmp_path / "outdir"), + get_data("tests/wf/" + test_file), + "--usermessage", + "hello", + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_issue_740_fixed(tmp_path: Path, factor: str) -> None: + """Confirm that re-running a particular workflow with caching succeeds.""" + test_file = "cache_test_workflow.cwl" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + commands = factor.split() + commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "Output of job will be cached in" not in stderr + assert error_code == 0, stderr + + +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_cache_relative_paths(tmp_path: Path, factor: str) -> None: + """Confirm that re-running a particular workflow with caching succeeds.""" + test_file = "secondary-files.cwl" + test_job_file = "secondary-files-job.yml" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend( + [ + "--out", + str(tmp_path / "out"), + "--cachedir", + cache_dir, + get_data(f"tests/{test_file}"), + get_data(f"tests/{test_job_file}"), + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + commands = factor.split() + commands.extend( + [ + "--out", + str(tmp_path / "out2"), + "--cachedir", + cache_dir, + get_data(f"tests/{test_file}"), + get_data(f"tests/{test_job_file}"), + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "Output of job will be cached in" not in stderr + assert error_code == 0, stderr + + assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists() + + +@pytest.mark.parametrize("factor", test_factors) +def test_cache_default_literal_file(tmp_path: Path, factor: str) -> None: + """Confirm that running a CLT with a default literal file with caching succeeds.""" + test_file = "tests/wf/extract_region_specs.cwl" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend( + [ + "--out", + str(tmp_path / "out"), + "--cachedir", + cache_dir, + get_data(test_file), + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_cache_dockerreq_hint_instead_of_req(tmp_path: Path, factor: str) -> None: + """The cache must not be checked when there is an invalid use of an absolute path in iwdr.listing.""" + cache_dir = str(tmp_path / "cwltool_cache") + test_job_file = "tests/wf/loadContents-input.yml" + # First, run the iwd-container-entryname1 conformance tests with caching turned on + test1_file = "tests/wf/iwd-container-entryname1.cwl" + commands1 = factor.split() + commands1.extend( + [ + "--out", + str(tmp_path / "out1"), + "--cachedir", + cache_dir, + get_data(test1_file), + get_data(test_job_file), + ] + ) + error_code1, _, stderr1 = get_main_output(commands1) + + stderr1 = re.sub(r"\s\s+", " ", stderr1) + assert "completed success" in stderr1 + assert error_code1 == 0 + # Second, run the iwd-container-entryname3 test, which should fail + # even though it would be a cache hit, except that its DockerRequirement is + # in `hints` instead of `requirements` and one of the initial working directory + # items has an absolute path starting with `/`. + test2_file = "tests/wf/iwd-container-entryname3.cwl" + commands2 = factor.split() + commands2.extend( + [ + "--out", + str(tmp_path / "out2"), + "--cachedir", + cache_dir, + get_data(test2_file), + get_data(test_job_file), + ] + ) + error_code2, _, stderr2 = get_main_output(commands2) + + stderr2 = re.sub(r"\s\s+", " ", stderr2) + assert ( + "at index 0 of listing is invalid, name can only start with '/' " + "when DockerRequirement is in 'requirements" in stderr2 + ) + assert error_code2 == 1 diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index ae18a41ae..f5ac0274b 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -6,7 +6,7 @@ from pathlib import Path from shutil import which from types import ModuleType -from typing import Optional, Tuple +from typing import Optional import pytest @@ -56,7 +56,7 @@ def test_biocontainers_resolution(tmp_path: Path) -> None: @pytest.fixture(scope="session") -def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]: +def bioconda_setup(request: pytest.FixtureRequest) -> tuple[Optional[int], str]: """ Caches the conda environment created for seqtk_seq.cwl. @@ -108,7 +108,7 @@ def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]: @pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed") -def test_bioconda(bioconda_setup: Tuple[Optional[int], str]) -> None: +def test_bioconda(bioconda_setup: tuple[Optional[int], str]) -> None: error_code, stderr = bioconda_setup assert error_code == 0, stderr @@ -119,15 +119,15 @@ def test_modules(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Do a basic smoke test using environment modules to satisfy a SoftwareRequirement.""" wflow = get_data("tests/random_lines.cwl") job = get_data("tests/random_lines_job.json") - monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles")) + monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles")) error_code, _, stderr = get_main_output( [ "--outdir", str(tmp_path / "out"), - "--beta-dependency-resolvers-configuration", "--beta-dependencies-directory", str(tmp_path / "deps"), - "tests/test_deps_env_modules_resolvers_conf.yml", + "--beta-dependency-resolvers-configuration", + get_data("tests/test_deps_env_modules_resolvers_conf.yml"), "--debug", wflow, job, @@ -145,7 +145,7 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Do so by by running `env` as the tool and parsing its output. """ - monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles")) + monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles")) tool_env = get_tool_env( tmp_path, [ @@ -155,6 +155,6 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> get_data("tests/env_with_software_req.yml"), ) - assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space " + assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space ", tool_env tool_path = tool_env["PATH"].split(":") assert get_data("tests/test_deps_env/random-lines/1.0/scripts") in tool_path diff --git a/tests/test_environment.py b/tests/test_environment.py index ba87041b3..fd2a160ec 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -2,10 +2,12 @@ import os from abc import ABC, abstractmethod +from collections.abc import Mapping from pathlib import Path -from typing import Any, Callable, Dict, List, Mapping, Union +from typing import Callable, Union import pytest +from packaging.version import Version from cwltool.singularity import get_version @@ -17,7 +19,7 @@ # TODO: maybe add regex? Env = Mapping[str, str] CheckerTypes = Union[None, str, Callable[[str, Env], bool]] -EnvChecks = Dict[str, CheckerTypes] +EnvChecks = dict[str, CheckerTypes] def assert_envvar_matches(check: CheckerTypes, k: str, env: Mapping[str, str]) -> None: @@ -66,7 +68,7 @@ def checks(tmp_prefix: str) -> EnvChecks: """Return a mapping from environment variable names to how to check for correctness.""" # Any flags to pass to cwltool to force use of the correct container - flags: List[str] + flags: list[str] # Does the env tool (maybe in our container) accept a `-0` flag? env_accepts_null: bool @@ -132,32 +134,33 @@ def PWD(v: str, env: Env) -> bool: } # Singularity variables appear to be in flux somewhat. - version = get_version()[0] - vmajor = version[0] - assert vmajor == 3, "Tests only work for Singularity 3" - vminor = version[1] + version = Version(".".join(map(str, get_version()[0]))) + assert version >= Version("3"), "Tests only work for Singularity 3+" sing_vars: EnvChecks = { "SINGULARITY_CONTAINER": None, "SINGULARITY_NAME": None, } - if vminor < 5: + if version < Version("3.5"): sing_vars["SINGULARITY_APPNAME"] = None - if vminor >= 5: + if (version >= Version("3.5")) and (version < Version("3.6")): + sing_vars["SINGULARITY_INIT"] = "1" + if version >= Version("3.5"): sing_vars["PROMPT_COMMAND"] = None sing_vars["SINGULARITY_ENVIRONMENT"] = None - if vminor == 5: - sing_vars["SINGULARITY_INIT"] = "1" - elif vminor > 5: + if version >= Version("3.6"): sing_vars["SINGULARITY_COMMAND"] = "exec" - if vminor >= 7: - if vminor > 9: - sing_vars["SINGULARITY_BIND"] = "" - else: + if version >= Version("3.7"): + if version > Version("3.9"): + sing_vars["SINGULARITY_BIND"] = "" + else: - def BIND(v: str, env: Env) -> bool: - return v.startswith(tmp_prefix) and v.endswith(":/tmp") + def BIND(v: str, env: Env) -> bool: + return v.startswith(tmp_prefix) and v.endswith(":/tmp") - sing_vars["SINGULARITY_BIND"] = BIND + sing_vars["SINGULARITY_BIND"] = BIND + if version >= Version("3.10"): + sing_vars["SINGULARITY_COMMAND"] = "run" + sing_vars["SINGULARITY_NO_EVAL"] = None result.update(sing_vars) @@ -194,14 +197,14 @@ def BIND(v: str, env: Env) -> bool: @CRT_PARAMS -def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> None: +def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test that basic env vars (only) show up.""" tmp_prefix = str(tmp_path / "canary") extra_env = { "USEDVAR": "VARVAL", "UNUSEDVAR": "VARVAL", } - args = crt_params.flags + [f"--tmpdir-prefix={tmp_prefix}"] + args = crt_params.flags + [f"--tmpdir-prefix={tmp_prefix}", "--debug"] env = get_tool_env( tmp_path, args, @@ -214,7 +217,9 @@ def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> Non @CRT_PARAMS -def test_preserve_single(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> None: +def test_preserve_single( + crt_params: CheckHolder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: """Test that preserving a single env var works.""" tmp_prefix = str(tmp_path / "canary") extra_env = { @@ -238,7 +243,9 @@ def test_preserve_single(crt_params: CheckHolder, tmp_path: Path, monkeypatch: A @CRT_PARAMS -def test_preserve_all(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> None: +def test_preserve_all( + crt_params: CheckHolder, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: """Test that preserving all works.""" tmp_prefix = str(tmp_path / "canary") extra_env = { diff --git a/tests/test_examples.py b/tests/test_examples.py index 4d479e313..0fe7f471c 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -9,7 +9,7 @@ import urllib.parse from io import StringIO from pathlib import Path -from typing import Any, Dict, List, Union, cast +from typing import Any, Union, cast import cwl_utils.expression as expr import pydot @@ -69,7 +69,7 @@ def test_expression_match(expression: str, expected: bool) -> None: assert (match is not None) == expected -interpolate_input = { +interpolate_input: dict[str, Any] = { "foo": { "bar": {"baz": "zab1"}, "b ar": {"baz": 2}, @@ -77,7 +77,7 @@ def test_expression_match(expression: str, expected: bool) -> None: 'b"ar': {"baz": None}, }, "lst": ["A", "B"], -} # type: Dict[str, Any] +} interpolate_parameters = [ ("$(foo)", interpolate_input["foo"]), @@ -410,7 +410,7 @@ def loadref( raise Exception("test case can't load things") scanned_deps = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( cast(str, obj["id"]), obj, @@ -473,7 +473,7 @@ def loadref( assert scanned_deps == expected_deps scanned_deps2 = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( cast(str, obj["id"]), obj, @@ -515,7 +515,7 @@ def loadref( raise Exception("test case can't load things") scanned_deps = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( "", obj, @@ -576,7 +576,7 @@ def test_scandeps_defaults_with_secondaryfiles() -> None: def test_dedupe() -> None: - not_deduped = [ + not_deduped: list[CWLObjectType] = [ {"class": "File", "location": "file:///example/a"}, {"class": "File", "location": "file:///example/a"}, {"class": "File", "location": "file:///example/d"}, @@ -585,7 +585,7 @@ def test_dedupe() -> None: "location": "file:///example/c", "listing": [{"class": "File", "location": "file:///example/d"}], }, - ] # type: List[CWLObjectType] + ] expected = [ {"class": "File", "location": "file:///example/a"}, @@ -649,7 +649,7 @@ def test_dedupe() -> None: @pytest.mark.parametrize("name, source, sink, expected", source_to_sink) def test_compare_types( - name: str, source: Dict[str, Any], sink: Dict[str, Any], expected: bool + name: str, source: dict[str, Any], sink: dict[str, Any], expected: bool ) -> None: assert can_assign_src_to_sink(source, sink) == expected, name @@ -675,7 +675,7 @@ def test_compare_types( @pytest.mark.parametrize("name, source, sink, expected", source_to_sink_strict) def test_compare_types_strict( - name: str, source: Dict[str, Any], sink: Dict[str, Any], expected: bool + name: str, source: dict[str, Any], sink: dict[str, Any], expected: bool ) -> None: assert can_assign_src_to_sink(source, sink, strict=True) == expected, name @@ -1118,7 +1118,7 @@ def test_cid_file_dir_arg_is_file_instead_of_dir(tmp_path: Path, factor: str) -> @needs_docker @pytest.mark.parametrize("factor", test_factors) def test_cid_file_non_existing_dir(tmp_path: Path, factor: str) -> None: - """Test that --cachedir with a bad path should produce a specific error.""" + """Test that --cidefile-dir with a bad path should produce a specific error.""" test_file = "cache_test_workflow.cwl" bad_cidfile_dir = tmp_path / "cidfile-dir-badpath" commands = factor.split() @@ -1233,120 +1233,6 @@ def test_secondary_files_v1_0(tmp_path: Path, factor: str) -> None: assert error_code == 0 -@needs_docker -@pytest.mark.parametrize("factor", test_factors) -def test_wf_without_container(tmp_path: Path, factor: str) -> None: - """Confirm that we can run a workflow without a container.""" - test_file = "hello-workflow.cwl" - cache_dir = str(tmp_path / "cwltool_cache") - commands = factor.split() - commands.extend( - [ - "--cachedir", - cache_dir, - "--outdir", - str(tmp_path / "outdir"), - get_data("tests/wf/" + test_file), - "--usermessage", - "hello", - ] - ) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "completed success" in stderr - assert error_code == 0 - - -@needs_docker -@pytest.mark.parametrize("factor", test_factors) -def test_issue_740_fixed(tmp_path: Path, factor: str) -> None: - """Confirm that re-running a particular workflow with caching succeeds.""" - test_file = "cache_test_workflow.cwl" - cache_dir = str(tmp_path / "cwltool_cache") - commands = factor.split() - commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "completed success" in stderr - assert error_code == 0 - - commands = factor.split() - commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "Output of job will be cached in" not in stderr - assert error_code == 0, stderr - - -@needs_docker -@pytest.mark.parametrize("factor", test_factors) -def test_cache_relative_paths(tmp_path: Path, factor: str) -> None: - """Confirm that re-running a particular workflow with caching succeeds.""" - test_file = "secondary-files.cwl" - test_job_file = "secondary-files-job.yml" - cache_dir = str(tmp_path / "cwltool_cache") - commands = factor.split() - commands.extend( - [ - "--out", - str(tmp_path / "out"), - "--cachedir", - cache_dir, - get_data(f"tests/{test_file}"), - get_data(f"tests/{test_job_file}"), - ] - ) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "completed success" in stderr - assert error_code == 0 - - commands = factor.split() - commands.extend( - [ - "--out", - str(tmp_path / "out2"), - "--cachedir", - cache_dir, - get_data(f"tests/{test_file}"), - get_data(f"tests/{test_job_file}"), - ] - ) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "Output of job will be cached in" not in stderr - assert error_code == 0, stderr - - assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists() - - -@pytest.mark.parametrize("factor", test_factors) -def test_cache_default_literal_file(tmp_path: Path, factor: str) -> None: - """Confirm that running a CLT with a default literal file with caching succeeds.""" - test_file = "tests/wf/extract_region_specs.cwl" - cache_dir = str(tmp_path / "cwltool_cache") - commands = factor.split() - commands.extend( - [ - "--out", - str(tmp_path / "out"), - "--cachedir", - cache_dir, - get_data(test_file), - ] - ) - error_code, _, stderr = get_main_output(commands) - - stderr = re.sub(r"\s\s+", " ", stderr) - assert "completed success" in stderr - assert error_code == 0 - - def test_write_summary(tmp_path: Path) -> None: """Test --write-summary.""" commands = [ @@ -1682,7 +1568,7 @@ def test_arguments_self() -> None: else: factory.runtime_context.use_container = False check = factory.make(get_data("tests/wf/paramref_arguments_self.cwl")) - outputs = cast(Dict[str, Any], check()) + outputs = cast(dict[str, Any], check()) assert "self_review" in outputs assert len(outputs) == 1 assert outputs["self_review"]["checksum"] == "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1" @@ -1820,9 +1706,9 @@ def test_validate_optional_src_with_mandatory_sink() -> None: ["--validate", get_data("tests/wf/optional_src_mandatory_sink.cwl")] ) assert exit_code == 0 - stderr = re.sub(r"\s\s+", " ", stderr) - assert 'Source \'opt_file\' of type ["null", "File"] may be incompatible' in stderr - assert "with sink 'r' of type \"File\"" in stderr + stdout = re.sub(r"\s\s+", " ", stdout) + assert 'Source \'opt_file\' of type ["null", "File"] may be incompatible' in stdout + assert "with sink 'r' of type \"File\"" in stdout def test_res_req_expr_float_1_0() -> None: @@ -1875,12 +1761,11 @@ def test_invalid_nested_array() -> None: ] ) assert exit_code == 1, stderr - stderr = re.sub(r"\n\s+", " ", stderr) - stderr = re.sub(r"\s\s+", " ", stderr) - assert "Tool definition failed validation:" in stderr + stdout = re.sub(r"\s\s+", " ", stdout) + assert "Tool definition failed validation:" in stdout assert ( - "tests/nested-array.cwl:6:5: Field 'type' references unknown identifier 'string[][]'" - ) in stderr + "tests/nested-array.cwl:7:5: Field 'type' references unknown identifier 'string[][]'" + ) in stdout def test_input_named_id() -> None: @@ -1894,3 +1779,15 @@ def test_input_named_id() -> None: ] ) assert exit_code == 0, stderr + + +def test_make_template() -> None: + """End-to-end test of --make-template, especially for mypyc mode.""" + exit_code, stdout, stderr = get_main_output( + [ + "--make-template", + "--debug", + get_data("tests/CometAdapter.cwl"), + ] + ) + assert exit_code == 0, stderr diff --git a/tests/test_fetch.py b/tests/test_fetch.py index e55491d90..962b7d7e5 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import Any, List, Optional +from typing import Any, Optional from urllib.parse import urljoin, urlsplit import pytest @@ -25,7 +25,7 @@ def __init__( ) -> None: """Create a Fetcher that provides a fixed result for testing purposes.""" - def fetch_text(self, url: str, content_types: Optional[List[str]] = None) -> str: + def fetch_text(self, url: str, content_types: Optional[list[str]] = None) -> str: if url == "baz:bar/foo.cwl": return """ cwlVersion: v1.0 diff --git a/tests/test_http_input.py b/tests/test_http_input.py index 6b4d9b479..e80260ff9 100644 --- a/tests/test_http_input.py +++ b/tests/test_http_input.py @@ -1,10 +1,7 @@ import os -import sys from datetime import datetime from pathlib import Path -from typing import List -import pytest from pytest_httpserver import HTTPServer from cwltool.pathmapper import PathMapper @@ -15,7 +12,7 @@ def test_http_path_mapping(tmp_path: Path) -> None: input_file_path = ( "https://raw.githubusercontent.com/common-workflow-language/cwltool/main/tests/2.fasta" ) - base_file: List[CWLObjectType] = [ + base_file: list[CWLObjectType] = [ { "class": "File", "location": "https://raw.githubusercontent.com/common-workflow-language/" @@ -34,7 +31,6 @@ def test_http_path_mapping(tmp_path: Path) -> None: assert ">Sequence 561 BP; 135 A; 106 C; 98 G; 222 T; 0 other;" in contents -@pytest.mark.skipif(sys.version_info < (3, 7), reason="timesout on CI") def test_modification_date(tmp_path: Path) -> None: """Local copies of remote files should preserve last modification date.""" # Initialize the server @@ -58,7 +54,7 @@ def test_modification_date(tmp_path: Path) -> None: ) location = httpserver.url_for(f"/{remote_file_name}") - base_file: List[CWLObjectType] = [ + base_file: list[CWLObjectType] = [ { "class": "File", "location": location, diff --git a/tests/test_js_sandbox.py b/tests/test_js_sandbox.py index f4839e8a0..2c5df6339 100644 --- a/tests/test_js_sandbox.py +++ b/tests/test_js_sandbox.py @@ -5,7 +5,7 @@ import shutil import threading from pathlib import Path -from typing import Any, List +from typing import Any import pytest from cwl_utils import sandboxjs @@ -48,8 +48,8 @@ def test_value_from_two_concatenated_expressions() -> None: def hide_nodejs(temp_dir: Path) -> str: """Generate a new PATH that hides node{js,}.""" - paths: List[str] = os.environ.get("PATH", "").split(":") - names: List[str] = [] + paths: list[str] = os.environ.get("PATH", "").split(":") + names: list[str] = [] for name in ("nodejs", "node"): path = shutil.which(name) if path: diff --git a/tests/test_loop.py b/tests/test_loop.py index bf908196d..e8a043611 100644 --- a/tests/test_loop.py +++ b/tests/test_loop.py @@ -1,8 +1,8 @@ """Test the 1.3 loop feature.""" import json +from collections.abc import MutableMapping, MutableSequence from io import StringIO -from typing import MutableMapping, MutableSequence from cwltool.main import main diff --git a/tests/test_loop_ext.py b/tests/test_loop_ext.py index 499dd17b4..1769d64ad 100644 --- a/tests/test_loop_ext.py +++ b/tests/test_loop_ext.py @@ -1,8 +1,8 @@ """Test the prototype cwltool:Loop extension.""" import json +from collections.abc import MutableMapping, MutableSequence from io import StringIO -from typing import MutableMapping, MutableSequence from cwltool.main import main diff --git a/tests/test_misc_cli.py b/tests/test_misc_cli.py index 307153e16..be314cad4 100644 --- a/tests/test_misc_cli.py +++ b/tests/test_misc_cli.py @@ -1,5 +1,7 @@ """Tests for various command line options.""" +import pytest + from cwltool.utils import versionstring from .util import get_data, get_main_output, needs_docker @@ -26,9 +28,13 @@ def test_empty_cmdling() -> None: assert "CWL document required, no input file was provided" in stderr -def test_tool_help() -> None: +def test_tool_help(monkeypatch: pytest.MonkeyPatch) -> None: """Test --tool-help.""" - return_code, stdout, stderr = get_main_output(["--tool-help", get_data("tests/echo.cwl")]) + return_code, stdout, stderr = get_main_output( + ["--tool-help", get_data("tests/echo.cwl")], + extra_env={"NO_COLOR": "1"}, + monkeypatch=monkeypatch, + ) assert return_code == 0 assert "job_order Job input json file" in stdout diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 643907a39..b36392a8a 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -3,23 +3,25 @@ import json import os.path import sys +from collections.abc import Generator, MutableMapping +from importlib.resources import files from io import StringIO from pathlib import Path -from typing import Any, Generator, List, MutableMapping, Optional, Tuple +from typing import Any, Optional, cast import pytest from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import Names +from schema_salad.ref_resolver import file_uri from schema_salad.utils import yaml_no_ts import cwltool.load_tool import cwltool.singularity import cwltool.udocker from cwltool.command_line_tool import CommandLineTool -from cwltool.context import LoadingContext, RuntimeContext +from cwltool.context import RuntimeContext from cwltool.main import main from cwltool.mpi import MpiConfig, MPIRequirementName -from cwltool.utils import files from .util import get_data, working_directory @@ -75,12 +77,12 @@ def __init__(self): else: self.indata = sys.stdin.read().encode(sys.stdin.encoding) - def run_once(self, args: List[str]): + def run_once(self, args: list[str]): subprocess.run( args, input=self.indata, stdout=sys.stdout, stderr=sys.stderr ).check_returncode() - def run_many(self, n: int, args: List[str]): + def run_many(self, n: int, args: list[str]): for i in range(n): self.run_once(args) @@ -122,7 +124,7 @@ def make_processes_input(np: int, tmp_path: Path) -> Path: return input_file -def cwltool_args(fake_mpi_conf: str) -> List[str]: +def cwltool_args(fake_mpi_conf: str) -> list[str]: return ["--enable-ext", "--enable-dev", "--mpi-config-file", fake_mpi_conf] @@ -291,15 +293,22 @@ def schema_ext11() -> Generator[Names, None, None]: mpiReq = CommentedMap({"class": MPIRequirementName, "processes": 1}) containerReq = CommentedMap({"class": "DockerRequirement"}) -basetool = CommentedMap({"cwlVersion": "v1.1", "inputs": CommentedSeq(), "outputs": CommentedSeq()}) +basetool = CommentedMap( + { + "cwlVersion": "v1.1", + "class": "CommandLineTool", + "inputs": CommentedSeq(), + "outputs": CommentedSeq(), + } +) def mk_tool( schema: Names, - opts: List[str], - reqs: Optional[List[CommentedMap]] = None, - hints: Optional[List[CommentedMap]] = None, -) -> Tuple[LoadingContext, RuntimeContext, CommentedMap]: + opts: list[str], + reqs: Optional[list[CommentedMap]] = None, + hints: Optional[list[CommentedMap]] = None, +) -> tuple[RuntimeContext, CommandLineTool]: tool = basetool.copy() if reqs is not None: @@ -309,53 +318,57 @@ def mk_tool( args = cwltool.argparser.arg_parser().parse_args(opts) args.enable_ext = True + args.basedir = os.path.dirname(os.path.abspath(".")) rc = RuntimeContext(vars(args)) lc = cwltool.main.setup_loadingContext(None, rc, args) lc.avsc_names = schema - return lc, rc, tool + tool["id"] = file_uri(os.path.abspath("./mktool.cwl")) + assert lc.loader is not None + lc.loader.idx[tool["id"]] = tool + return rc, cast(CommandLineTool, cwltool.load_tool.load_tool(tool, lc)) def test_singularity(schema_ext11: Names) -> None: - lc, rc, tool = mk_tool(schema_ext11, ["--singularity"], reqs=[mpiReq, containerReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, ["--singularity"], reqs=[mpiReq, containerReq]) + clt._init_job({}, rc) jr = clt.make_job_runner(rc) assert jr is cwltool.singularity.SingularityCommandLineJob def test_udocker(schema_ext11: Names) -> None: - lc, rc, tool = mk_tool(schema_ext11, ["--udocker"], reqs=[mpiReq, containerReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, ["--udocker"], reqs=[mpiReq, containerReq]) + clt._init_job({}, rc) jr = clt.make_job_runner(rc) assert jr is cwltool.udocker.UDockerCommandLineJob def test_docker_hint(schema_ext11: Names) -> None: # Docker hint, MPI required - lc, rc, tool = mk_tool(schema_ext11, [], hints=[containerReq], reqs=[mpiReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, [], hints=[containerReq], reqs=[mpiReq]) + clt._init_job({}, rc) jr = clt.make_job_runner(rc) assert jr is cwltool.job.CommandLineJob def test_docker_required(schema_ext11: Names) -> None: # Docker required, MPI hinted - lc, rc, tool = mk_tool(schema_ext11, [], reqs=[containerReq], hints=[mpiReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, [], reqs=[containerReq], hints=[mpiReq]) + clt._init_job({}, rc) jr = clt.make_job_runner(rc) assert jr is cwltool.docker.DockerCommandLineJob def test_docker_mpi_both_required(schema_ext11: Names) -> None: # Both required - error - lc, rc, tool = mk_tool(schema_ext11, [], reqs=[mpiReq, containerReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, [], reqs=[mpiReq, containerReq]) with pytest.raises(cwltool.errors.UnsupportedRequirement): - clt.make_job_runner(rc) + clt._init_job({}, rc) + clt.make_job_runner(rc) def test_docker_mpi_both_hinted(schema_ext11: Names) -> None: # Both hinted - error - lc, rc, tool = mk_tool(schema_ext11, [], hints=[mpiReq, containerReq]) - clt = CommandLineTool(tool, lc) + rc, clt = mk_tool(schema_ext11, [], hints=[mpiReq, containerReq]) with pytest.raises(cwltool.errors.UnsupportedRequirement): - clt.make_job_runner(rc) + clt._init_job({}, rc) + clt.make_job_runner(rc) diff --git a/tests/test_override.py b/tests/test_override.py index 980c853bb..93c836c84 100644 --- a/tests/test_override.py +++ b/tests/test_override.py @@ -1,6 +1,5 @@ import json from io import StringIO -from typing import Dict, List import pytest @@ -76,7 +75,7 @@ @needs_docker @pytest.mark.parametrize("parameters,result", override_parameters) -def test_overrides(parameters: List[str], result: Dict[str, str]) -> None: +def test_overrides(parameters: list[str], result: dict[str, str]) -> None: sio = StringIO() assert main(parameters, stdout=sio) == 0 @@ -119,7 +118,7 @@ def test_overrides(parameters: List[str], result: Dict[str, str]) -> None: @needs_docker @pytest.mark.parametrize("parameters,expected_error", failing_override_parameters) -def test_overrides_fails(parameters: List[str], expected_error: str) -> None: +def test_overrides_fails(parameters: list[str], expected_error: str) -> None: sio = StringIO() assert main(parameters, stderr=sio) == 1 diff --git a/tests/test_pack.py b/tests/test_pack.py index 1d38e35e8..a65996f8f 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -5,7 +5,6 @@ from functools import partial from io import StringIO from pathlib import Path -from typing import Dict import pytest from schema_salad.utils import yaml_no_ts @@ -95,7 +94,7 @@ def test_pack_fragment() -> None: def test_pack_rewrites() -> None: - rewrites: Dict[str, str] = {} + rewrites: dict[str, str] = {} loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/default-wf5.cwl")) loadingContext.do_update = False diff --git a/tests/test_path_checks.py b/tests/test_path_checks.py index 01ab7fe17..096de9942 100644 --- a/tests/test_path_checks.py +++ b/tests/test_path_checks.py @@ -1,7 +1,7 @@ import urllib.parse from io import BytesIO from pathlib import Path -from typing import IO, Any, List, cast +from typing import IO, Any, cast import pytest from ruamel.yaml.comments import CommentedMap @@ -112,7 +112,7 @@ def test_unicode_in_output_files(tmp_path: Path, filename: str) -> None: class StubFsAccess(StdFsAccess): """Stub fs access object that doesn't rely on the filesystem.""" - def glob(self, pattern: str) -> List[str]: + def glob(self, pattern: str) -> list[str]: """glob.""" return [pattern] diff --git a/tests/test_pathmapper.py b/tests/test_pathmapper.py index b7cf2f6a1..4ffac24bd 100644 --- a/tests/test_pathmapper.py +++ b/tests/test_pathmapper.py @@ -1,5 +1,3 @@ -from typing import List, Tuple - import pytest from cwltool.pathmapper import PathMapper @@ -10,7 +8,7 @@ def test_subclass() -> None: class SubPathMapper(PathMapper): def __init__( self, - referenced_files: List[CWLObjectType], + referenced_files: list[CWLObjectType], basedir: str, stagedir: str, new: str, @@ -81,7 +79,7 @@ def test_normalizeFilesDirs(name: str, file_dir: CWLObjectType, expected: CWLObj @pytest.mark.parametrize("filename,expected", basename_generation_parameters) -def test_basename_field_generation(filename: str, expected: Tuple[str, str]) -> None: +def test_basename_field_generation(filename: str, expected: tuple[str, str]) -> None: nameroot, nameext = expected expected2 = { "class": "File", diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 83eb61c22..d7a2a698b 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -3,8 +3,9 @@ import pickle import sys import urllib +from collections.abc import Generator from pathlib import Path -from typing import IO, Any, Generator, cast +from typing import IO, Any, cast import arcp import bagit @@ -31,12 +32,23 @@ SCHEMA = Namespace("http://schema.org/") CWLPROV = Namespace("https://w3id.org/cwl/prov#") OA = Namespace("http://www.w3.org/ns/oa#") +FOAF = Namespace("http://xmlns.com/foaf/0.1/") -def cwltool(tmp_path: Path, *args: Any) -> Path: +TEST_ORCID = "https://orcid.org/0000-0003-4862-3349" + + +def cwltool(tmp_path: Path, *args: Any, with_orcid: bool = False) -> Path: prov_folder = tmp_path / "provenance" prov_folder.mkdir() - new_args = ["--provenance", str(prov_folder)] + new_args = [ + "--enable-user-provenance", + "--enable-host-provenance", + "--provenance", + str(prov_folder), + ] + if with_orcid: + new_args.extend(["--orcid", TEST_ORCID]) new_args.extend(args) # Run within a temporary directory to not pollute git checkout tmp_dir = tmp_path / "cwltool-run" @@ -48,61 +60,81 @@ def cwltool(tmp_path: Path, *args: Any) -> Path: @needs_docker -def test_hello_workflow(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_hello_workflow(tmp_path: Path, with_orcid: bool) -> None: check_provenance( cwltool( tmp_path, get_data("tests/wf/hello-workflow.cwl"), "--usermessage", "Hello workflow", - ) + with_orcid=with_orcid, + ), + with_orcid=with_orcid, ) @needs_docker -def test_hello_single_tool(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_hello_single_tool(tmp_path: Path, with_orcid: bool) -> None: check_provenance( cwltool( tmp_path, get_data("tests/wf/hello_single_tool.cwl"), "--message", "Hello tool", + with_orcid=with_orcid, ), single_tool=True, + with_orcid=with_orcid, ) @needs_docker -def test_revsort_workflow(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_revsort_workflow(tmp_path: Path, with_orcid: bool) -> None: folder = cwltool( tmp_path, get_data("tests/wf/revsort.cwl"), get_data("tests/wf/revsort-job.json"), + with_orcid=with_orcid, ) check_output_object(folder) - check_provenance(folder) + check_provenance(folder, with_orcid=with_orcid) @needs_docker -def test_revsort_workflow_shortcut(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_revsort_workflow_shortcut(tmp_path: Path, with_orcid: bool) -> None: """Confirm that using 'cwl:tool' shortcut still snapshots the CWL files.""" folder = cwltool( tmp_path, get_data("tests/wf/revsort-job-shortcut.json"), + with_orcid=with_orcid, ) check_output_object(folder) - check_provenance(folder) + check_provenance(folder, with_orcid=with_orcid) assert not (folder / "snapshot" / "revsort-job-shortcut.json").exists() assert len(list((folder / "snapshot").iterdir())) == 4 @needs_docker -def test_nested_workflow(tmp_path: Path) -> None: - check_provenance(cwltool(tmp_path, get_data("tests/wf/nested.cwl")), nested=True) +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_nested_workflow(tmp_path: Path, with_orcid: bool) -> None: + check_provenance( + cwltool( + tmp_path, + get_data("tests/wf/nested.cwl"), + with_orcid=with_orcid, + ), + nested=True, + with_orcid=with_orcid, + ) @needs_docker -def test_secondary_files_implicit(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_secondary_files_implicit(tmp_path: Path, with_orcid: bool) -> None: file1 = tmp_path / "foo1.txt" file1idx = tmp_path / "foo1.txt.idx" @@ -112,13 +144,20 @@ def test_secondary_files_implicit(tmp_path: Path) -> None: f.write("bar") # secondary will be picked up by .idx - folder = cwltool(tmp_path, get_data("tests/wf/sec-wf.cwl"), "--file1", str(file1)) - check_provenance(folder, secondary_files=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/sec-wf.cwl"), + "--file1", + str(file1), + with_orcid=with_orcid, + ) + check_provenance(folder, secondary_files=True, with_orcid=with_orcid) check_secondary_files(folder) @needs_docker -def test_secondary_files_explicit(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_secondary_files_explicit(tmp_path: Path, with_orcid: bool) -> None: # Deliberately do NOT have common basename or extension file1dir = tmp_path / "foo" file1dir.mkdir() @@ -153,22 +192,33 @@ def test_secondary_files_explicit(tmp_path: Path) -> None: j = json.dumps(job, ensure_ascii=True) fp.write(j.encode("ascii")) - folder = cwltool(tmp_path, get_data("tests/wf/sec-wf.cwl"), str(jobJson)) - check_provenance(folder, secondary_files=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/sec-wf.cwl"), + str(jobJson), + with_orcid=with_orcid, + ) + check_provenance(folder, secondary_files=True, with_orcid=with_orcid) check_secondary_files(folder) @needs_docker -def test_secondary_files_output(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_secondary_files_output(tmp_path: Path, with_orcid: bool) -> None: # secondary will be picked up by .idx - folder = cwltool(tmp_path, get_data("tests/wf/sec-wf-out.cwl")) - check_provenance(folder, secondary_files=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/sec-wf-out.cwl"), + with_orcid=with_orcid, + ) + check_provenance(folder, secondary_files=True, with_orcid=with_orcid) # Skipped, not the same secondary files as above # self.check_secondary_files() @needs_docker -def test_directory_workflow(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_directory_workflow(tmp_path: Path, with_orcid: bool) -> None: dir2 = tmp_path / "dir2" dir2.mkdir() sha1 = { @@ -184,8 +234,14 @@ def test_directory_workflow(tmp_path: Path) -> None: with open(dir2 / x, "w", encoding="ascii") as f: f.write(x) - folder = cwltool(tmp_path, get_data("tests/wf/directory.cwl"), "--dir", str(dir2)) - check_provenance(folder, directory=True) + folder = cwltool( + tmp_path, + get_data("tests/wf/directory.cwl"), + "--dir", + str(dir2), + with_orcid=with_orcid, + ) + check_provenance(folder, directory=True, with_orcid=with_orcid) # Output should include ls stdout of filenames a b c on each line file_list = ( @@ -208,10 +264,12 @@ def test_directory_workflow(tmp_path: Path) -> None: @needs_docker -def test_no_data_files(tmp_path: Path) -> None: +@pytest.mark.parametrize("with_orcid", [True, False]) +def test_no_data_files(tmp_path: Path, with_orcid: bool) -> None: folder = cwltool( tmp_path, get_data("tests/wf/conditional_step_no_inputs.cwl"), + with_orcid=with_orcid, ) check_bagit(folder) @@ -262,6 +320,7 @@ def check_provenance( single_tool: bool = False, directory: bool = False, secondary_files: bool = False, + with_orcid: bool = False, ) -> None: check_folders(base_path) check_bagit(base_path) @@ -272,6 +331,7 @@ def check_provenance( single_tool=single_tool, directory=directory, secondary_files=secondary_files, + with_orcid=with_orcid, ) @@ -462,6 +522,7 @@ def check_prov( single_tool: bool = False, directory: bool = False, secondary_files: bool = False, + with_orcid: bool = False, ) -> None: prov_file = base_path / "metadata" / "provenance" / "primary.cwlprov.nt" assert prov_file.is_file(), f"Can't find {prov_file}" @@ -484,7 +545,6 @@ def check_prov( # the has_provenance annotations in manifest.json instead # run should have been started by a wf engine - engines = set(g.subjects(RDF.type, WFPROV.WorkflowEngine)) assert engines, "Could not find WorkflowEngine" assert len(engines) == 1, "Found too many WorkflowEngines: %s" % engines @@ -501,6 +561,39 @@ def check_prov( PROV.SoftwareAgent, ) in g, "Engine not declared as SoftwareAgent" + # run should be associated to the user + accounts = set(g.subjects(RDF.type, FOAF.OnlineAccount)) + assert len(accounts) == 1 + account = accounts.pop() + people = set(g.subjects(RDF.type, SCHEMA.Person)) + assert len(people) == 1, "Can't find associated person in workflow run" + person = people.pop() + if with_orcid: + assert person == URIRef(TEST_ORCID) + else: + account_names = set(g.objects(account, FOAF.accountName)) + assert len(account_names) == 1 + account_name = cast(Literal, account_names.pop()) + machine_user = provenance._whoami()[0] + assert account_name.value == machine_user + + # find the random UUID assigned to cwltool + tool_agents = set(g.subjects(RDF.type, PROV.SoftwareAgent)) + n_all_agents = 2 + len(tool_agents) + agents = set(g.subjects(RDF.type, PROV.Agent)) + assert ( + len(agents) == n_all_agents + ), "There should be 1 agent per tool (engine), 1 user agent, and 1 cwltool agent" + agents.remove(person) + agents.remove(engine) # the main tool + remain_agents = agents - tool_agents + assert len(remain_agents) == 1 + assert ( + account, + PROV.actedOnBehalfOf, + person, + ) in g, "Association of cwltool agent acting for user is missing" + if single_tool: activities = set(g.subjects(RDF.type, PROV.Activity)) assert len(activities) == 1, "Too many activities: %s" % activities diff --git a/tests/test_relocate.py b/tests/test_relocate.py index 81877c776..692e995fa 100644 --- a/tests/test_relocate.py +++ b/tests/test_relocate.py @@ -1,18 +1,13 @@ import json import os import shutil -import sys +from io import StringIO from pathlib import Path from cwltool.main import main from .util import get_data, needs_docker -if sys.version_info[0] < 3: - from StringIO import StringIO -else: - from io import StringIO - @needs_docker def test_for_910(tmp_path: Path) -> None: diff --git a/tests/test_secrets.py b/tests/test_secrets.py index bd90bee78..a8c0b67af 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -1,7 +1,7 @@ import shutil import tempfile from io import StringIO -from typing import Callable, Dict, List, Tuple, Union +from typing import Callable, Union import pytest @@ -13,7 +13,7 @@ @pytest.fixture -def secrets() -> Tuple[SecretStore, CWLObjectType]: +def secrets() -> tuple[SecretStore, CWLObjectType]: """Fixture to return a secret store.""" sec_store = SecretStore() job: CWLObjectType = {"foo": "bar", "baz": "quux"} @@ -22,7 +22,7 @@ def secrets() -> Tuple[SecretStore, CWLObjectType]: return sec_store, job -def test_obscuring(secrets: Tuple[SecretStore, CWLObjectType]) -> None: +def test_obscuring(secrets: tuple[SecretStore, CWLObjectType]) -> None: """Basic test of secret store.""" storage, obscured = secrets assert obscured["foo"] != "bar" @@ -41,8 +41,8 @@ def test_obscuring(secrets: Tuple[SecretStore, CWLObjectType]) -> None: @pytest.mark.parametrize("factory,expected", obscured_factories_expected) def test_secrets( factory: Callable[[str], CWLObjectType], - expected: Union[str, List[str], Dict[str, str]], - secrets: Tuple[SecretStore, CWLObjectType], + expected: Union[str, list[str], dict[str, str]], + secrets: tuple[SecretStore, CWLObjectType], ) -> None: storage, obscured = secrets obs = obscured["foo"] diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 0512f2e28..1139dfbc7 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -2,7 +2,6 @@ import shutil from pathlib import Path -from typing import Any import pytest @@ -19,7 +18,7 @@ @needs_singularity_2_6 -def test_singularity_pullfolder(tmp_path: Path, monkeypatch: Any) -> None: +def test_singularity_pullfolder(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test singularity respects SINGULARITY_PULLFOLDER.""" workdir = tmp_path / "working_dir_new" workdir.mkdir() diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index 73fe240d0..18a588cf8 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -6,7 +6,7 @@ import subprocess import sys from pathlib import Path -from typing import List, cast +from typing import cast import pytest from ruamel.yaml.comments import CommentedMap @@ -318,7 +318,7 @@ def test_docker_tmpdir_prefix(tmp_path: Path) -> None: "docker", ) job = DockerCommandLineJob(builder, {}, CommandLineTool.make_path_mapper, [], [], "") - runtime: List[str] = [] + runtime: list[str] = [] volume_writable_file = MapperEnt( resolved=get_data("tests/2.fastq"), target="foo", type=None, staged=None diff --git a/tests/test_toolargparse.py b/tests/test_toolargparse.py index 11ce5e3db..2e50fe722 100644 --- a/tests/test_toolargparse.py +++ b/tests/test_toolargparse.py @@ -1,7 +1,7 @@ import argparse from io import StringIO from pathlib import Path -from typing import Callable, List +from typing import Callable import pytest @@ -296,7 +296,7 @@ def test_argparser_without_doc() -> None: ), ], ) -def test_argparse_append_with_default(job_order: List[str], expected_values: List[str]) -> None: +def test_argparse_append_with_default(job_order: list[str], expected_values: list[str]) -> None: """ Confirm that the appended arguments must not include the default. diff --git a/tests/test_validate.py b/tests/test_validate.py index 171a6b6c1..8e664d9aa 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -1,5 +1,7 @@ """Tests --validation.""" +import io +import logging import re from .util import get_data, get_main_output @@ -43,13 +45,83 @@ def test_validate_with_invalid_input_object() -> None: ] ) assert exit_code == 1 - stderr = re.sub(r"\s\s+", " ", stderr) - assert "Invalid job input record" in stderr + stdout = re.sub(r"\s\s+", " ", stdout) + assert "Invalid job input record" in stdout assert ( "tests/wf/1st-workflow_bad_inputs.yml:2:1: * the 'ex' field is not " - "valid because the value is not string" in stderr + "valid because the value is not string" in stdout ) assert ( "tests/wf/1st-workflow_bad_inputs.yml:1:1: * the 'inp' field is not " - "valid because is not a dict. Expected a File object." in stderr + "valid because is not a dict. Expected a File object." in stdout + ) + + +def test_validate_quiet() -> None: + """Ensure that --validate --quiet prints the correct amount of information.""" + exit_code, stdout, stderr = get_main_output( + [ + "--validate", + "--quiet", + get_data("tests/CometAdapter.cwl"), + ] + ) + assert exit_code == 0 + stdout = re.sub(r"\s\s+", " ", stdout) + assert "INFO" not in stdout + assert "INFO" not in stderr + assert "tests/CometAdapter.cwl:10:3: object id" in stdout + assert "tests/CometAdapter.cwl#out' previously defined" in stdout + + +def test_validate_no_warnings() -> None: + """Ensure that --validate --no-warnings doesn't print any warnings.""" + exit_code, stdout, stderr = get_main_output( + [ + "--validate", + "--no-warnings", + get_data("tests/CometAdapter.cwl"), + ] ) + assert exit_code == 0 + stdout = re.sub(r"\s\s+", " ", stdout) + stderr = re.sub(r"\s\s+", " ", stderr) + assert "INFO" not in stdout + assert "INFO" not in stderr + assert "WARNING" not in stdout + assert "WARNING" not in stderr + assert "tests/CometAdapter.cwl:9:3: object id" not in stdout + assert "tests/CometAdapter.cwl:9:3: object id" not in stderr + assert "tests/CometAdapter.cwl#out' previously defined" not in stdout + assert "tests/CometAdapter.cwl#out' previously defined" not in stderr + + +def test_validate_custom_logger() -> None: + """Custom log handling test.""" + custom_log = io.StringIO() + handler = logging.StreamHandler(custom_log) + handler.setLevel(logging.DEBUG) + exit_code, stdout, stderr = get_main_output( + [ + "--validate", + get_data("tests/CometAdapter.cwl"), + ], + logger_handler=handler, + ) + custom_log_text = custom_log.getvalue() + assert exit_code == 0 + custom_log_text = re.sub(r"\s\s+", " ", custom_log_text) + stdout = re.sub(r"\s\s+", " ", stdout) + stderr = re.sub(r"\s\s+", " ", stderr) + assert "INFO" not in stdout + assert "INFO" not in stderr + assert "INFO" in custom_log_text + assert "WARNING" not in stdout + assert "WARNING" not in stderr + assert "WARNING" in custom_log_text + assert "tests/CometAdapter.cwl:10:3: object id" not in stdout + assert "tests/CometAdapter.cwl:10:3: object id" not in stderr + assert "tests/CometAdapter.cwl:10:3: object id" in custom_log_text + assert "tests/CometAdapter.cwl#out' previously defined" not in stdout + assert "tests/CometAdapter.cwl#out' previously defined" not in stderr + assert "tests/CometAdapter.cwl#out' previously defined" in custom_log_text diff --git a/tests/trs/Dockstore.cwl b/tests/trs/Dockstore.cwl old mode 100644 new mode 100755 diff --git a/tests/trs/md5sum-tool.cwl b/tests/trs/md5sum-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/trs/md5sum-workflow.cwl b/tests/trs/md5sum-workflow.cwl old mode 100644 new mode 100755 diff --git a/tests/utf_doc_example.cwl b/tests/utf_doc_example.cwl old mode 100644 new mode 100755 diff --git a/tests/util.py b/tests/util.py index 0547cfa9a..d7624bc5e 100644 --- a/tests/util.py +++ b/tests/util.py @@ -8,16 +8,17 @@ import shutil import subprocess import sys +from collections.abc import Generator, Mapping from contextlib import ExitStack +from importlib.resources import as_file, files from pathlib import Path -from typing import Dict, Generator, List, Mapping, Optional, Tuple, Union +from typing import Any, Optional, Union import pytest from cwltool.env_to_stdout import deserialize_env from cwltool.main import main from cwltool.singularity import is_version_2_6, is_version_3_or_newer -from cwltool.utils import as_file, files def force_default_container(default_container_id: str, _: str) -> str: @@ -83,11 +84,12 @@ def env_accepts_null() -> bool: def get_main_output( - args: List[str], + args: list[str], replacement_env: Optional[Mapping[str, str]] = None, extra_env: Optional[Mapping[str, str]] = None, monkeypatch: Optional[pytest.MonkeyPatch] = None, -) -> Tuple[Optional[int], str, str]: + **extra_kwargs: Any, +) -> tuple[Optional[int], str, str]: """Run cwltool main. args: the command line args to call it with @@ -112,7 +114,7 @@ def get_main_output( monkeypatch.setenv(k, v) try: - rc = main(argsl=args, stdout=stdout, stderr=stderr) + rc = main(argsl=args, stdout=stdout, stderr=stderr, **extra_kwargs) except SystemExit as e: if isinstance(e.code, int): rc = e.code @@ -127,13 +129,13 @@ def get_main_output( def get_tool_env( tmp_path: Path, - flag_args: List[str], + flag_args: list[str], inputs_file: Optional[str] = None, replacement_env: Optional[Mapping[str, str]] = None, extra_env: Optional[Mapping[str, str]] = None, monkeypatch: Optional[pytest.MonkeyPatch] = None, runtime_env_accepts_null: Optional[bool] = None, -) -> Dict[str, str]: +) -> dict[str, str]: """Get the env vars for a tool's invocation.""" # GNU env accepts the -0 option to end each variable's # printing with "\0". No such luck on BSD-ish. diff --git a/tests/wc-tool-bad-hints.cwl b/tests/wc-tool-bad-hints.cwl old mode 100644 new mode 100755 diff --git a/tests/wc-tool-bad-reqs.cwl b/tests/wc-tool-bad-reqs.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/1496.cwl b/tests/wf/1496.cwl old mode 100644 new mode 100755 index 74f6ef49e..d5da025dd --- a/tests/wf/1496.cwl +++ b/tests/wf/1496.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool diff --git a/tests/wf/1590.cwl b/tests/wf/1590.cwl old mode 100644 new mode 100755 index 19c8ad331..de602a500 --- a/tests/wf/1590.cwl +++ b/tests/wf/1590.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner { "baseCommand": [ "cat" diff --git a/tests/wf/1st-workflow.cwl b/tests/wf/1st-workflow.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/811-12.cwl b/tests/wf/811-12.cwl old mode 100644 new mode 100755 index f4403f45b..ab0b2bd92 --- a/tests/wf/811-12.cwl +++ b/tests/wf/811-12.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: Workflow diff --git a/tests/wf/811.cwl b/tests/wf/811.cwl old mode 100644 new mode 100755 index 16a4c828d..8ef826fdc --- a/tests/wf/811.cwl +++ b/tests/wf/811.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: Workflow diff --git a/tests/wf/816_tool.cwl b/tests/wf/816_tool.cwl old mode 100644 new mode 100755 index 00395abe8..c48d3e373 --- a/tests/wf/816_tool.cwl +++ b/tests/wf/816_tool.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: CommandLineTool requirements: diff --git a/tests/wf/816_wf.cwl b/tests/wf/816_wf.cwl old mode 100644 new mode 100755 index a64b130b6..9db143916 --- a/tests/wf/816_wf.cwl +++ b/tests/wf/816_wf.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: Workflow cwlVersion: v1.0 inputs: diff --git a/tests/wf/910.cwl b/tests/wf/910.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/arguments.cwl b/tests/wf/arguments.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad-stderr-expr.cwl b/tests/wf/bad-stderr-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad-stdin-expr.cwl b/tests/wf/bad-stdin-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad-stdout-expr.cwl b/tests/wf/bad-stdout-expr.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad_networkaccess.cwl b/tests/wf/bad_networkaccess.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/bad_timelimit.cwl b/tests/wf/bad_timelimit.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/cache_test_workflow.cwl b/tests/wf/cache_test_workflow.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/cat-tool.cwl b/tests/wf/cat-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/conditional_step_no_inputs.cwl b/tests/wf/conditional_step_no_inputs.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/conflict.cwl b/tests/wf/conflict.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/cores_float.cwl b/tests/wf/cores_float.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/default-dir5.cwl b/tests/wf/default-dir5.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/default-wf5.cwl b/tests/wf/default-wf5.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/double-nested.cwl b/tests/wf/double-nested.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/expect_trick_packed.cwl b/tests/wf/expect_trick_packed.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/extract_region_specs.cwl b/tests/wf/extract_region_specs.cwl old mode 100644 new mode 100755 index 279fa4400..437222a13 --- a/tests/wf/extract_region_specs.cwl +++ b/tests/wf/extract_region_specs.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner { "cwlVersion": "v1.0", "class": "CommandLineTool", diff --git a/tests/wf/floats_small_and_large.cwl b/tests/wf/floats_small_and_large.cwl old mode 100644 new mode 100755 index 434327361..1637bd9af --- a/tests/wf/floats_small_and_large.cwl +++ b/tests/wf/floats_small_and_large.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: CommandLineTool baseCommand: echo diff --git a/tests/wf/generator/pytoolgen.cwl b/tests/wf/generator/pytoolgen.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/inp-filelist.txt b/tests/wf/inp-filelist.txt new file mode 100644 index 000000000..232ddf670 --- /dev/null +++ b/tests/wf/inp-filelist.txt @@ -0,0 +1,9999 @@ +example_input_file1.txt +example_input_file2.txt +example_input_file3.txt +example_input_file4.txt +example_input_file5.txt +example_input_file6.txt +example_input_file7.txt +example_input_file8.txt +example_input_file9.txt +example_input_file10.txt +example_input_file11.txt +example_input_file12.txt +example_input_file13.txt +example_input_file14.txt +example_input_file15.txt +example_input_file16.txt +example_input_file17.txt +example_input_file18.txt +example_input_file19.txt +example_input_file20.txt +example_input_file21.txt +example_input_file22.txt +example_input_file23.txt +example_input_file24.txt +example_input_file25.txt +example_input_file26.txt +example_input_file27.txt +example_input_file28.txt +example_input_file29.txt +example_input_file30.txt +example_input_file31.txt +example_input_file32.txt +example_input_file33.txt +example_input_file34.txt +example_input_file35.txt +example_input_file36.txt +example_input_file37.txt +example_input_file38.txt +example_input_file39.txt +example_input_file40.txt +example_input_file41.txt +example_input_file42.txt +example_input_file43.txt +example_input_file44.txt +example_input_file45.txt +example_input_file46.txt +example_input_file47.txt +example_input_file48.txt +example_input_file49.txt +example_input_file50.txt +example_input_file51.txt +example_input_file52.txt +example_input_file53.txt +example_input_file54.txt +example_input_file55.txt +example_input_file56.txt +example_input_file57.txt +example_input_file58.txt +example_input_file59.txt +example_input_file60.txt +example_input_file61.txt +example_input_file62.txt +example_input_file63.txt +example_input_file64.txt +example_input_file65.txt +example_input_file66.txt +example_input_file67.txt +example_input_file68.txt +example_input_file69.txt +example_input_file70.txt +example_input_file71.txt +example_input_file72.txt +example_input_file73.txt +example_input_file74.txt +example_input_file75.txt +example_input_file76.txt +example_input_file77.txt +example_input_file78.txt +example_input_file79.txt +example_input_file80.txt +example_input_file81.txt +example_input_file82.txt +example_input_file83.txt +example_input_file84.txt +example_input_file85.txt +example_input_file86.txt +example_input_file87.txt +example_input_file88.txt +example_input_file89.txt +example_input_file90.txt +example_input_file91.txt +example_input_file92.txt +example_input_file93.txt +example_input_file94.txt +example_input_file95.txt +example_input_file96.txt +example_input_file97.txt +example_input_file98.txt +example_input_file99.txt +example_input_file100.txt +example_input_file101.txt +example_input_file102.txt +example_input_file103.txt +example_input_file104.txt +example_input_file105.txt +example_input_file106.txt +example_input_file107.txt +example_input_file108.txt +example_input_file109.txt +example_input_file110.txt +example_input_file111.txt +example_input_file112.txt +example_input_file113.txt +example_input_file114.txt +example_input_file115.txt +example_input_file116.txt +example_input_file117.txt +example_input_file118.txt +example_input_file119.txt +example_input_file120.txt +example_input_file121.txt +example_input_file122.txt +example_input_file123.txt +example_input_file124.txt +example_input_file125.txt +example_input_file126.txt +example_input_file127.txt +example_input_file128.txt +example_input_file129.txt +example_input_file130.txt +example_input_file131.txt +example_input_file132.txt +example_input_file133.txt +example_input_file134.txt +example_input_file135.txt +example_input_file136.txt +example_input_file137.txt +example_input_file138.txt +example_input_file139.txt +example_input_file140.txt +example_input_file141.txt +example_input_file142.txt +example_input_file143.txt +example_input_file144.txt +example_input_file145.txt +example_input_file146.txt +example_input_file147.txt +example_input_file148.txt +example_input_file149.txt +example_input_file150.txt +example_input_file151.txt +example_input_file152.txt +example_input_file153.txt +example_input_file154.txt +example_input_file155.txt +example_input_file156.txt +example_input_file157.txt +example_input_file158.txt +example_input_file159.txt +example_input_file160.txt +example_input_file161.txt +example_input_file162.txt +example_input_file163.txt +example_input_file164.txt +example_input_file165.txt +example_input_file166.txt +example_input_file167.txt +example_input_file168.txt +example_input_file169.txt +example_input_file170.txt +example_input_file171.txt +example_input_file172.txt +example_input_file173.txt +example_input_file174.txt +example_input_file175.txt +example_input_file176.txt +example_input_file177.txt +example_input_file178.txt +example_input_file179.txt +example_input_file180.txt +example_input_file181.txt +example_input_file182.txt +example_input_file183.txt +example_input_file184.txt +example_input_file185.txt +example_input_file186.txt +example_input_file187.txt +example_input_file188.txt +example_input_file189.txt +example_input_file190.txt +example_input_file191.txt +example_input_file192.txt +example_input_file193.txt +example_input_file194.txt +example_input_file195.txt +example_input_file196.txt +example_input_file197.txt +example_input_file198.txt +example_input_file199.txt +example_input_file200.txt +example_input_file201.txt +example_input_file202.txt +example_input_file203.txt +example_input_file204.txt +example_input_file205.txt +example_input_file206.txt +example_input_file207.txt +example_input_file208.txt +example_input_file209.txt +example_input_file210.txt +example_input_file211.txt +example_input_file212.txt +example_input_file213.txt +example_input_file214.txt +example_input_file215.txt +example_input_file216.txt +example_input_file217.txt +example_input_file218.txt +example_input_file219.txt +example_input_file220.txt +example_input_file221.txt +example_input_file222.txt +example_input_file223.txt +example_input_file224.txt +example_input_file225.txt +example_input_file226.txt +example_input_file227.txt +example_input_file228.txt +example_input_file229.txt +example_input_file230.txt +example_input_file231.txt +example_input_file232.txt +example_input_file233.txt +example_input_file234.txt +example_input_file235.txt +example_input_file236.txt +example_input_file237.txt +example_input_file238.txt +example_input_file239.txt +example_input_file240.txt +example_input_file241.txt +example_input_file242.txt +example_input_file243.txt +example_input_file244.txt +example_input_file245.txt +example_input_file246.txt +example_input_file247.txt +example_input_file248.txt +example_input_file249.txt +example_input_file250.txt +example_input_file251.txt +example_input_file252.txt +example_input_file253.txt +example_input_file254.txt +example_input_file255.txt +example_input_file256.txt +example_input_file257.txt +example_input_file258.txt +example_input_file259.txt +example_input_file260.txt +example_input_file261.txt +example_input_file262.txt +example_input_file263.txt +example_input_file264.txt +example_input_file265.txt +example_input_file266.txt +example_input_file267.txt +example_input_file268.txt +example_input_file269.txt +example_input_file270.txt +example_input_file271.txt +example_input_file272.txt +example_input_file273.txt +example_input_file274.txt +example_input_file275.txt +example_input_file276.txt +example_input_file277.txt +example_input_file278.txt +example_input_file279.txt +example_input_file280.txt +example_input_file281.txt +example_input_file282.txt +example_input_file283.txt +example_input_file284.txt +example_input_file285.txt +example_input_file286.txt +example_input_file287.txt +example_input_file288.txt +example_input_file289.txt +example_input_file290.txt +example_input_file291.txt +example_input_file292.txt +example_input_file293.txt +example_input_file294.txt +example_input_file295.txt +example_input_file296.txt +example_input_file297.txt +example_input_file298.txt +example_input_file299.txt +example_input_file300.txt +example_input_file301.txt +example_input_file302.txt +example_input_file303.txt +example_input_file304.txt +example_input_file305.txt +example_input_file306.txt +example_input_file307.txt +example_input_file308.txt +example_input_file309.txt +example_input_file310.txt +example_input_file311.txt +example_input_file312.txt +example_input_file313.txt +example_input_file314.txt +example_input_file315.txt +example_input_file316.txt +example_input_file317.txt +example_input_file318.txt +example_input_file319.txt +example_input_file320.txt +example_input_file321.txt +example_input_file322.txt +example_input_file323.txt +example_input_file324.txt +example_input_file325.txt +example_input_file326.txt +example_input_file327.txt +example_input_file328.txt +example_input_file329.txt +example_input_file330.txt +example_input_file331.txt +example_input_file332.txt +example_input_file333.txt +example_input_file334.txt +example_input_file335.txt +example_input_file336.txt +example_input_file337.txt +example_input_file338.txt +example_input_file339.txt +example_input_file340.txt +example_input_file341.txt +example_input_file342.txt +example_input_file343.txt +example_input_file344.txt +example_input_file345.txt +example_input_file346.txt +example_input_file347.txt +example_input_file348.txt +example_input_file349.txt +example_input_file350.txt +example_input_file351.txt +example_input_file352.txt +example_input_file353.txt +example_input_file354.txt +example_input_file355.txt +example_input_file356.txt +example_input_file357.txt +example_input_file358.txt +example_input_file359.txt +example_input_file360.txt +example_input_file361.txt +example_input_file362.txt +example_input_file363.txt +example_input_file364.txt +example_input_file365.txt +example_input_file366.txt +example_input_file367.txt +example_input_file368.txt +example_input_file369.txt +example_input_file370.txt +example_input_file371.txt +example_input_file372.txt +example_input_file373.txt +example_input_file374.txt +example_input_file375.txt +example_input_file376.txt +example_input_file377.txt +example_input_file378.txt +example_input_file379.txt +example_input_file380.txt +example_input_file381.txt +example_input_file382.txt +example_input_file383.txt +example_input_file384.txt +example_input_file385.txt +example_input_file386.txt +example_input_file387.txt +example_input_file388.txt +example_input_file389.txt +example_input_file390.txt +example_input_file391.txt +example_input_file392.txt +example_input_file393.txt +example_input_file394.txt +example_input_file395.txt +example_input_file396.txt +example_input_file397.txt +example_input_file398.txt +example_input_file399.txt +example_input_file400.txt +example_input_file401.txt +example_input_file402.txt +example_input_file403.txt +example_input_file404.txt +example_input_file405.txt +example_input_file406.txt +example_input_file407.txt +example_input_file408.txt +example_input_file409.txt +example_input_file410.txt +example_input_file411.txt +example_input_file412.txt +example_input_file413.txt +example_input_file414.txt +example_input_file415.txt +example_input_file416.txt +example_input_file417.txt +example_input_file418.txt +example_input_file419.txt +example_input_file420.txt +example_input_file421.txt +example_input_file422.txt +example_input_file423.txt +example_input_file424.txt +example_input_file425.txt +example_input_file426.txt +example_input_file427.txt +example_input_file428.txt +example_input_file429.txt +example_input_file430.txt +example_input_file431.txt +example_input_file432.txt +example_input_file433.txt +example_input_file434.txt +example_input_file435.txt +example_input_file436.txt +example_input_file437.txt +example_input_file438.txt +example_input_file439.txt +example_input_file440.txt +example_input_file441.txt +example_input_file442.txt +example_input_file443.txt +example_input_file444.txt +example_input_file445.txt +example_input_file446.txt +example_input_file447.txt +example_input_file448.txt +example_input_file449.txt +example_input_file450.txt +example_input_file451.txt +example_input_file452.txt +example_input_file453.txt +example_input_file454.txt +example_input_file455.txt +example_input_file456.txt +example_input_file457.txt +example_input_file458.txt +example_input_file459.txt +example_input_file460.txt +example_input_file461.txt +example_input_file462.txt +example_input_file463.txt +example_input_file464.txt +example_input_file465.txt +example_input_file466.txt +example_input_file467.txt +example_input_file468.txt +example_input_file469.txt +example_input_file470.txt +example_input_file471.txt +example_input_file472.txt +example_input_file473.txt +example_input_file474.txt +example_input_file475.txt +example_input_file476.txt +example_input_file477.txt +example_input_file478.txt +example_input_file479.txt +example_input_file480.txt +example_input_file481.txt +example_input_file482.txt +example_input_file483.txt +example_input_file484.txt +example_input_file485.txt +example_input_file486.txt +example_input_file487.txt +example_input_file488.txt +example_input_file489.txt +example_input_file490.txt +example_input_file491.txt +example_input_file492.txt +example_input_file493.txt +example_input_file494.txt +example_input_file495.txt +example_input_file496.txt +example_input_file497.txt +example_input_file498.txt +example_input_file499.txt +example_input_file500.txt +example_input_file501.txt +example_input_file502.txt +example_input_file503.txt +example_input_file504.txt +example_input_file505.txt +example_input_file506.txt +example_input_file507.txt +example_input_file508.txt +example_input_file509.txt +example_input_file510.txt +example_input_file511.txt +example_input_file512.txt +example_input_file513.txt +example_input_file514.txt +example_input_file515.txt +example_input_file516.txt +example_input_file517.txt +example_input_file518.txt +example_input_file519.txt +example_input_file520.txt +example_input_file521.txt +example_input_file522.txt +example_input_file523.txt +example_input_file524.txt +example_input_file525.txt +example_input_file526.txt +example_input_file527.txt +example_input_file528.txt +example_input_file529.txt +example_input_file530.txt +example_input_file531.txt +example_input_file532.txt +example_input_file533.txt +example_input_file534.txt +example_input_file535.txt +example_input_file536.txt +example_input_file537.txt +example_input_file538.txt +example_input_file539.txt +example_input_file540.txt +example_input_file541.txt +example_input_file542.txt +example_input_file543.txt +example_input_file544.txt +example_input_file545.txt +example_input_file546.txt +example_input_file547.txt +example_input_file548.txt +example_input_file549.txt +example_input_file550.txt +example_input_file551.txt +example_input_file552.txt +example_input_file553.txt +example_input_file554.txt +example_input_file555.txt +example_input_file556.txt +example_input_file557.txt +example_input_file558.txt +example_input_file559.txt +example_input_file560.txt +example_input_file561.txt +example_input_file562.txt +example_input_file563.txt +example_input_file564.txt +example_input_file565.txt +example_input_file566.txt +example_input_file567.txt +example_input_file568.txt +example_input_file569.txt +example_input_file570.txt +example_input_file571.txt +example_input_file572.txt +example_input_file573.txt +example_input_file574.txt +example_input_file575.txt +example_input_file576.txt +example_input_file577.txt +example_input_file578.txt +example_input_file579.txt +example_input_file580.txt +example_input_file581.txt +example_input_file582.txt +example_input_file583.txt +example_input_file584.txt +example_input_file585.txt +example_input_file586.txt +example_input_file587.txt +example_input_file588.txt +example_input_file589.txt +example_input_file590.txt +example_input_file591.txt +example_input_file592.txt +example_input_file593.txt +example_input_file594.txt +example_input_file595.txt +example_input_file596.txt +example_input_file597.txt +example_input_file598.txt +example_input_file599.txt +example_input_file600.txt +example_input_file601.txt +example_input_file602.txt +example_input_file603.txt +example_input_file604.txt +example_input_file605.txt +example_input_file606.txt +example_input_file607.txt +example_input_file608.txt +example_input_file609.txt +example_input_file610.txt +example_input_file611.txt +example_input_file612.txt +example_input_file613.txt +example_input_file614.txt +example_input_file615.txt +example_input_file616.txt +example_input_file617.txt +example_input_file618.txt +example_input_file619.txt +example_input_file620.txt +example_input_file621.txt +example_input_file622.txt +example_input_file623.txt +example_input_file624.txt +example_input_file625.txt +example_input_file626.txt +example_input_file627.txt +example_input_file628.txt +example_input_file629.txt +example_input_file630.txt +example_input_file631.txt +example_input_file632.txt +example_input_file633.txt +example_input_file634.txt +example_input_file635.txt +example_input_file636.txt +example_input_file637.txt +example_input_file638.txt +example_input_file639.txt +example_input_file640.txt +example_input_file641.txt +example_input_file642.txt +example_input_file643.txt +example_input_file644.txt +example_input_file645.txt +example_input_file646.txt +example_input_file647.txt +example_input_file648.txt +example_input_file649.txt +example_input_file650.txt +example_input_file651.txt +example_input_file652.txt +example_input_file653.txt +example_input_file654.txt +example_input_file655.txt +example_input_file656.txt +example_input_file657.txt +example_input_file658.txt +example_input_file659.txt +example_input_file660.txt +example_input_file661.txt +example_input_file662.txt +example_input_file663.txt +example_input_file664.txt +example_input_file665.txt +example_input_file666.txt +example_input_file667.txt +example_input_file668.txt +example_input_file669.txt +example_input_file670.txt +example_input_file671.txt +example_input_file672.txt +example_input_file673.txt +example_input_file674.txt +example_input_file675.txt +example_input_file676.txt +example_input_file677.txt +example_input_file678.txt +example_input_file679.txt +example_input_file680.txt +example_input_file681.txt +example_input_file682.txt +example_input_file683.txt +example_input_file684.txt +example_input_file685.txt +example_input_file686.txt +example_input_file687.txt +example_input_file688.txt +example_input_file689.txt +example_input_file690.txt +example_input_file691.txt +example_input_file692.txt +example_input_file693.txt +example_input_file694.txt +example_input_file695.txt +example_input_file696.txt +example_input_file697.txt +example_input_file698.txt +example_input_file699.txt +example_input_file700.txt +example_input_file701.txt +example_input_file702.txt +example_input_file703.txt +example_input_file704.txt +example_input_file705.txt +example_input_file706.txt +example_input_file707.txt +example_input_file708.txt +example_input_file709.txt +example_input_file710.txt +example_input_file711.txt +example_input_file712.txt +example_input_file713.txt +example_input_file714.txt +example_input_file715.txt +example_input_file716.txt +example_input_file717.txt +example_input_file718.txt +example_input_file719.txt +example_input_file720.txt +example_input_file721.txt +example_input_file722.txt +example_input_file723.txt +example_input_file724.txt +example_input_file725.txt +example_input_file726.txt +example_input_file727.txt +example_input_file728.txt +example_input_file729.txt +example_input_file730.txt +example_input_file731.txt +example_input_file732.txt +example_input_file733.txt +example_input_file734.txt +example_input_file735.txt +example_input_file736.txt +example_input_file737.txt +example_input_file738.txt +example_input_file739.txt +example_input_file740.txt +example_input_file741.txt +example_input_file742.txt +example_input_file743.txt +example_input_file744.txt +example_input_file745.txt +example_input_file746.txt +example_input_file747.txt +example_input_file748.txt +example_input_file749.txt +example_input_file750.txt +example_input_file751.txt +example_input_file752.txt +example_input_file753.txt +example_input_file754.txt +example_input_file755.txt +example_input_file756.txt +example_input_file757.txt +example_input_file758.txt +example_input_file759.txt +example_input_file760.txt +example_input_file761.txt +example_input_file762.txt +example_input_file763.txt +example_input_file764.txt +example_input_file765.txt +example_input_file766.txt +example_input_file767.txt +example_input_file768.txt +example_input_file769.txt +example_input_file770.txt +example_input_file771.txt +example_input_file772.txt +example_input_file773.txt +example_input_file774.txt +example_input_file775.txt +example_input_file776.txt +example_input_file777.txt +example_input_file778.txt +example_input_file779.txt +example_input_file780.txt +example_input_file781.txt +example_input_file782.txt +example_input_file783.txt +example_input_file784.txt +example_input_file785.txt +example_input_file786.txt +example_input_file787.txt +example_input_file788.txt +example_input_file789.txt +example_input_file790.txt +example_input_file791.txt +example_input_file792.txt +example_input_file793.txt +example_input_file794.txt +example_input_file795.txt +example_input_file796.txt +example_input_file797.txt +example_input_file798.txt +example_input_file799.txt +example_input_file800.txt +example_input_file801.txt +example_input_file802.txt +example_input_file803.txt +example_input_file804.txt +example_input_file805.txt +example_input_file806.txt +example_input_file807.txt +example_input_file808.txt +example_input_file809.txt +example_input_file810.txt +example_input_file811.txt +example_input_file812.txt +example_input_file813.txt +example_input_file814.txt +example_input_file815.txt +example_input_file816.txt +example_input_file817.txt +example_input_file818.txt +example_input_file819.txt +example_input_file820.txt +example_input_file821.txt +example_input_file822.txt +example_input_file823.txt +example_input_file824.txt +example_input_file825.txt +example_input_file826.txt +example_input_file827.txt +example_input_file828.txt +example_input_file829.txt +example_input_file830.txt +example_input_file831.txt +example_input_file832.txt +example_input_file833.txt +example_input_file834.txt +example_input_file835.txt +example_input_file836.txt +example_input_file837.txt +example_input_file838.txt +example_input_file839.txt +example_input_file840.txt +example_input_file841.txt +example_input_file842.txt +example_input_file843.txt +example_input_file844.txt +example_input_file845.txt +example_input_file846.txt +example_input_file847.txt +example_input_file848.txt +example_input_file849.txt +example_input_file850.txt +example_input_file851.txt +example_input_file852.txt +example_input_file853.txt +example_input_file854.txt +example_input_file855.txt +example_input_file856.txt +example_input_file857.txt +example_input_file858.txt +example_input_file859.txt +example_input_file860.txt +example_input_file861.txt +example_input_file862.txt +example_input_file863.txt +example_input_file864.txt +example_input_file865.txt +example_input_file866.txt +example_input_file867.txt +example_input_file868.txt +example_input_file869.txt +example_input_file870.txt +example_input_file871.txt +example_input_file872.txt +example_input_file873.txt +example_input_file874.txt +example_input_file875.txt +example_input_file876.txt +example_input_file877.txt +example_input_file878.txt +example_input_file879.txt +example_input_file880.txt +example_input_file881.txt +example_input_file882.txt +example_input_file883.txt +example_input_file884.txt +example_input_file885.txt +example_input_file886.txt +example_input_file887.txt +example_input_file888.txt +example_input_file889.txt +example_input_file890.txt +example_input_file891.txt +example_input_file892.txt +example_input_file893.txt +example_input_file894.txt +example_input_file895.txt +example_input_file896.txt +example_input_file897.txt +example_input_file898.txt +example_input_file899.txt +example_input_file900.txt +example_input_file901.txt +example_input_file902.txt +example_input_file903.txt +example_input_file904.txt +example_input_file905.txt +example_input_file906.txt +example_input_file907.txt +example_input_file908.txt +example_input_file909.txt +example_input_file910.txt +example_input_file911.txt +example_input_file912.txt +example_input_file913.txt +example_input_file914.txt +example_input_file915.txt +example_input_file916.txt +example_input_file917.txt +example_input_file918.txt +example_input_file919.txt +example_input_file920.txt +example_input_file921.txt +example_input_file922.txt +example_input_file923.txt +example_input_file924.txt +example_input_file925.txt +example_input_file926.txt +example_input_file927.txt +example_input_file928.txt +example_input_file929.txt +example_input_file930.txt +example_input_file931.txt +example_input_file932.txt +example_input_file933.txt +example_input_file934.txt +example_input_file935.txt +example_input_file936.txt +example_input_file937.txt +example_input_file938.txt +example_input_file939.txt +example_input_file940.txt +example_input_file941.txt +example_input_file942.txt +example_input_file943.txt +example_input_file944.txt +example_input_file945.txt +example_input_file946.txt +example_input_file947.txt +example_input_file948.txt +example_input_file949.txt +example_input_file950.txt +example_input_file951.txt +example_input_file952.txt +example_input_file953.txt +example_input_file954.txt +example_input_file955.txt +example_input_file956.txt +example_input_file957.txt +example_input_file958.txt +example_input_file959.txt +example_input_file960.txt +example_input_file961.txt +example_input_file962.txt +example_input_file963.txt +example_input_file964.txt +example_input_file965.txt +example_input_file966.txt +example_input_file967.txt +example_input_file968.txt +example_input_file969.txt +example_input_file970.txt +example_input_file971.txt +example_input_file972.txt +example_input_file973.txt +example_input_file974.txt +example_input_file975.txt +example_input_file976.txt +example_input_file977.txt +example_input_file978.txt +example_input_file979.txt +example_input_file980.txt +example_input_file981.txt +example_input_file982.txt +example_input_file983.txt +example_input_file984.txt +example_input_file985.txt +example_input_file986.txt +example_input_file987.txt +example_input_file988.txt +example_input_file989.txt +example_input_file990.txt +example_input_file991.txt +example_input_file992.txt +example_input_file993.txt +example_input_file994.txt +example_input_file995.txt +example_input_file996.txt +example_input_file997.txt +example_input_file998.txt +example_input_file999.txt +example_input_file1000.txt +example_input_file1001.txt +example_input_file1002.txt +example_input_file1003.txt +example_input_file1004.txt +example_input_file1005.txt +example_input_file1006.txt +example_input_file1007.txt +example_input_file1008.txt +example_input_file1009.txt +example_input_file1010.txt +example_input_file1011.txt +example_input_file1012.txt +example_input_file1013.txt +example_input_file1014.txt +example_input_file1015.txt +example_input_file1016.txt +example_input_file1017.txt +example_input_file1018.txt +example_input_file1019.txt +example_input_file1020.txt +example_input_file1021.txt +example_input_file1022.txt +example_input_file1023.txt +example_input_file1024.txt +example_input_file1025.txt +example_input_file1026.txt +example_input_file1027.txt +example_input_file1028.txt +example_input_file1029.txt +example_input_file1030.txt +example_input_file1031.txt +example_input_file1032.txt +example_input_file1033.txt +example_input_file1034.txt +example_input_file1035.txt +example_input_file1036.txt +example_input_file1037.txt +example_input_file1038.txt +example_input_file1039.txt +example_input_file1040.txt +example_input_file1041.txt +example_input_file1042.txt +example_input_file1043.txt +example_input_file1044.txt +example_input_file1045.txt +example_input_file1046.txt +example_input_file1047.txt +example_input_file1048.txt +example_input_file1049.txt +example_input_file1050.txt +example_input_file1051.txt +example_input_file1052.txt +example_input_file1053.txt +example_input_file1054.txt +example_input_file1055.txt +example_input_file1056.txt +example_input_file1057.txt +example_input_file1058.txt +example_input_file1059.txt +example_input_file1060.txt +example_input_file1061.txt +example_input_file1062.txt +example_input_file1063.txt +example_input_file1064.txt +example_input_file1065.txt +example_input_file1066.txt +example_input_file1067.txt +example_input_file1068.txt +example_input_file1069.txt +example_input_file1070.txt +example_input_file1071.txt +example_input_file1072.txt +example_input_file1073.txt +example_input_file1074.txt +example_input_file1075.txt +example_input_file1076.txt +example_input_file1077.txt +example_input_file1078.txt +example_input_file1079.txt +example_input_file1080.txt +example_input_file1081.txt +example_input_file1082.txt +example_input_file1083.txt +example_input_file1084.txt +example_input_file1085.txt +example_input_file1086.txt +example_input_file1087.txt +example_input_file1088.txt +example_input_file1089.txt +example_input_file1090.txt +example_input_file1091.txt +example_input_file1092.txt +example_input_file1093.txt +example_input_file1094.txt +example_input_file1095.txt +example_input_file1096.txt +example_input_file1097.txt +example_input_file1098.txt +example_input_file1099.txt +example_input_file1100.txt +example_input_file1101.txt +example_input_file1102.txt +example_input_file1103.txt +example_input_file1104.txt +example_input_file1105.txt +example_input_file1106.txt +example_input_file1107.txt +example_input_file1108.txt +example_input_file1109.txt +example_input_file1110.txt +example_input_file1111.txt +example_input_file1112.txt +example_input_file1113.txt +example_input_file1114.txt +example_input_file1115.txt +example_input_file1116.txt +example_input_file1117.txt +example_input_file1118.txt +example_input_file1119.txt +example_input_file1120.txt +example_input_file1121.txt +example_input_file1122.txt +example_input_file1123.txt +example_input_file1124.txt +example_input_file1125.txt +example_input_file1126.txt +example_input_file1127.txt +example_input_file1128.txt +example_input_file1129.txt +example_input_file1130.txt +example_input_file1131.txt +example_input_file1132.txt +example_input_file1133.txt +example_input_file1134.txt +example_input_file1135.txt +example_input_file1136.txt +example_input_file1137.txt +example_input_file1138.txt +example_input_file1139.txt +example_input_file1140.txt +example_input_file1141.txt +example_input_file1142.txt +example_input_file1143.txt +example_input_file1144.txt +example_input_file1145.txt +example_input_file1146.txt +example_input_file1147.txt +example_input_file1148.txt +example_input_file1149.txt +example_input_file1150.txt +example_input_file1151.txt +example_input_file1152.txt +example_input_file1153.txt +example_input_file1154.txt +example_input_file1155.txt +example_input_file1156.txt +example_input_file1157.txt +example_input_file1158.txt +example_input_file1159.txt +example_input_file1160.txt +example_input_file1161.txt +example_input_file1162.txt +example_input_file1163.txt +example_input_file1164.txt +example_input_file1165.txt +example_input_file1166.txt +example_input_file1167.txt +example_input_file1168.txt +example_input_file1169.txt +example_input_file1170.txt +example_input_file1171.txt +example_input_file1172.txt +example_input_file1173.txt +example_input_file1174.txt +example_input_file1175.txt +example_input_file1176.txt +example_input_file1177.txt +example_input_file1178.txt +example_input_file1179.txt +example_input_file1180.txt +example_input_file1181.txt +example_input_file1182.txt +example_input_file1183.txt +example_input_file1184.txt +example_input_file1185.txt +example_input_file1186.txt +example_input_file1187.txt +example_input_file1188.txt +example_input_file1189.txt +example_input_file1190.txt +example_input_file1191.txt +example_input_file1192.txt +example_input_file1193.txt +example_input_file1194.txt +example_input_file1195.txt +example_input_file1196.txt +example_input_file1197.txt +example_input_file1198.txt +example_input_file1199.txt +example_input_file1200.txt +example_input_file1201.txt +example_input_file1202.txt +example_input_file1203.txt +example_input_file1204.txt +example_input_file1205.txt +example_input_file1206.txt +example_input_file1207.txt +example_input_file1208.txt +example_input_file1209.txt +example_input_file1210.txt +example_input_file1211.txt +example_input_file1212.txt +example_input_file1213.txt +example_input_file1214.txt +example_input_file1215.txt +example_input_file1216.txt +example_input_file1217.txt +example_input_file1218.txt +example_input_file1219.txt +example_input_file1220.txt +example_input_file1221.txt +example_input_file1222.txt +example_input_file1223.txt +example_input_file1224.txt +example_input_file1225.txt +example_input_file1226.txt +example_input_file1227.txt +example_input_file1228.txt +example_input_file1229.txt +example_input_file1230.txt +example_input_file1231.txt +example_input_file1232.txt +example_input_file1233.txt +example_input_file1234.txt +example_input_file1235.txt +example_input_file1236.txt +example_input_file1237.txt +example_input_file1238.txt +example_input_file1239.txt +example_input_file1240.txt +example_input_file1241.txt +example_input_file1242.txt +example_input_file1243.txt +example_input_file1244.txt +example_input_file1245.txt +example_input_file1246.txt +example_input_file1247.txt +example_input_file1248.txt +example_input_file1249.txt +example_input_file1250.txt +example_input_file1251.txt +example_input_file1252.txt +example_input_file1253.txt +example_input_file1254.txt +example_input_file1255.txt +example_input_file1256.txt +example_input_file1257.txt +example_input_file1258.txt +example_input_file1259.txt +example_input_file1260.txt +example_input_file1261.txt +example_input_file1262.txt +example_input_file1263.txt +example_input_file1264.txt +example_input_file1265.txt +example_input_file1266.txt +example_input_file1267.txt +example_input_file1268.txt +example_input_file1269.txt +example_input_file1270.txt +example_input_file1271.txt +example_input_file1272.txt +example_input_file1273.txt +example_input_file1274.txt +example_input_file1275.txt +example_input_file1276.txt +example_input_file1277.txt +example_input_file1278.txt +example_input_file1279.txt +example_input_file1280.txt +example_input_file1281.txt +example_input_file1282.txt +example_input_file1283.txt +example_input_file1284.txt +example_input_file1285.txt +example_input_file1286.txt +example_input_file1287.txt +example_input_file1288.txt +example_input_file1289.txt +example_input_file1290.txt +example_input_file1291.txt +example_input_file1292.txt +example_input_file1293.txt +example_input_file1294.txt +example_input_file1295.txt +example_input_file1296.txt +example_input_file1297.txt +example_input_file1298.txt +example_input_file1299.txt +example_input_file1300.txt +example_input_file1301.txt +example_input_file1302.txt +example_input_file1303.txt +example_input_file1304.txt +example_input_file1305.txt +example_input_file1306.txt +example_input_file1307.txt +example_input_file1308.txt +example_input_file1309.txt +example_input_file1310.txt +example_input_file1311.txt +example_input_file1312.txt +example_input_file1313.txt +example_input_file1314.txt +example_input_file1315.txt +example_input_file1316.txt +example_input_file1317.txt +example_input_file1318.txt +example_input_file1319.txt +example_input_file1320.txt +example_input_file1321.txt +example_input_file1322.txt +example_input_file1323.txt +example_input_file1324.txt +example_input_file1325.txt +example_input_file1326.txt +example_input_file1327.txt +example_input_file1328.txt +example_input_file1329.txt +example_input_file1330.txt +example_input_file1331.txt +example_input_file1332.txt +example_input_file1333.txt +example_input_file1334.txt +example_input_file1335.txt +example_input_file1336.txt +example_input_file1337.txt +example_input_file1338.txt +example_input_file1339.txt +example_input_file1340.txt +example_input_file1341.txt +example_input_file1342.txt +example_input_file1343.txt +example_input_file1344.txt +example_input_file1345.txt +example_input_file1346.txt +example_input_file1347.txt +example_input_file1348.txt +example_input_file1349.txt +example_input_file1350.txt +example_input_file1351.txt +example_input_file1352.txt +example_input_file1353.txt +example_input_file1354.txt +example_input_file1355.txt +example_input_file1356.txt +example_input_file1357.txt +example_input_file1358.txt +example_input_file1359.txt +example_input_file1360.txt +example_input_file1361.txt +example_input_file1362.txt +example_input_file1363.txt +example_input_file1364.txt +example_input_file1365.txt +example_input_file1366.txt +example_input_file1367.txt +example_input_file1368.txt +example_input_file1369.txt +example_input_file1370.txt +example_input_file1371.txt +example_input_file1372.txt +example_input_file1373.txt +example_input_file1374.txt +example_input_file1375.txt +example_input_file1376.txt +example_input_file1377.txt +example_input_file1378.txt +example_input_file1379.txt +example_input_file1380.txt +example_input_file1381.txt +example_input_file1382.txt +example_input_file1383.txt +example_input_file1384.txt +example_input_file1385.txt +example_input_file1386.txt +example_input_file1387.txt +example_input_file1388.txt +example_input_file1389.txt +example_input_file1390.txt +example_input_file1391.txt +example_input_file1392.txt +example_input_file1393.txt +example_input_file1394.txt +example_input_file1395.txt +example_input_file1396.txt +example_input_file1397.txt +example_input_file1398.txt +example_input_file1399.txt +example_input_file1400.txt +example_input_file1401.txt +example_input_file1402.txt +example_input_file1403.txt +example_input_file1404.txt +example_input_file1405.txt +example_input_file1406.txt +example_input_file1407.txt +example_input_file1408.txt +example_input_file1409.txt +example_input_file1410.txt +example_input_file1411.txt +example_input_file1412.txt +example_input_file1413.txt +example_input_file1414.txt +example_input_file1415.txt +example_input_file1416.txt +example_input_file1417.txt +example_input_file1418.txt +example_input_file1419.txt +example_input_file1420.txt +example_input_file1421.txt +example_input_file1422.txt +example_input_file1423.txt +example_input_file1424.txt +example_input_file1425.txt +example_input_file1426.txt +example_input_file1427.txt +example_input_file1428.txt +example_input_file1429.txt +example_input_file1430.txt +example_input_file1431.txt +example_input_file1432.txt +example_input_file1433.txt +example_input_file1434.txt +example_input_file1435.txt +example_input_file1436.txt +example_input_file1437.txt +example_input_file1438.txt +example_input_file1439.txt +example_input_file1440.txt +example_input_file1441.txt +example_input_file1442.txt +example_input_file1443.txt +example_input_file1444.txt +example_input_file1445.txt +example_input_file1446.txt +example_input_file1447.txt +example_input_file1448.txt +example_input_file1449.txt +example_input_file1450.txt +example_input_file1451.txt +example_input_file1452.txt +example_input_file1453.txt +example_input_file1454.txt +example_input_file1455.txt +example_input_file1456.txt +example_input_file1457.txt +example_input_file1458.txt +example_input_file1459.txt +example_input_file1460.txt +example_input_file1461.txt +example_input_file1462.txt +example_input_file1463.txt +example_input_file1464.txt +example_input_file1465.txt +example_input_file1466.txt +example_input_file1467.txt +example_input_file1468.txt +example_input_file1469.txt +example_input_file1470.txt +example_input_file1471.txt +example_input_file1472.txt +example_input_file1473.txt +example_input_file1474.txt +example_input_file1475.txt +example_input_file1476.txt +example_input_file1477.txt +example_input_file1478.txt +example_input_file1479.txt +example_input_file1480.txt +example_input_file1481.txt +example_input_file1482.txt +example_input_file1483.txt +example_input_file1484.txt +example_input_file1485.txt +example_input_file1486.txt +example_input_file1487.txt +example_input_file1488.txt +example_input_file1489.txt +example_input_file1490.txt +example_input_file1491.txt +example_input_file1492.txt +example_input_file1493.txt +example_input_file1494.txt +example_input_file1495.txt +example_input_file1496.txt +example_input_file1497.txt +example_input_file1498.txt +example_input_file1499.txt +example_input_file1500.txt +example_input_file1501.txt +example_input_file1502.txt +example_input_file1503.txt +example_input_file1504.txt +example_input_file1505.txt +example_input_file1506.txt +example_input_file1507.txt +example_input_file1508.txt +example_input_file1509.txt +example_input_file1510.txt +example_input_file1511.txt +example_input_file1512.txt +example_input_file1513.txt +example_input_file1514.txt +example_input_file1515.txt +example_input_file1516.txt +example_input_file1517.txt +example_input_file1518.txt +example_input_file1519.txt +example_input_file1520.txt +example_input_file1521.txt +example_input_file1522.txt +example_input_file1523.txt +example_input_file1524.txt +example_input_file1525.txt +example_input_file1526.txt +example_input_file1527.txt +example_input_file1528.txt +example_input_file1529.txt +example_input_file1530.txt +example_input_file1531.txt +example_input_file1532.txt +example_input_file1533.txt +example_input_file1534.txt +example_input_file1535.txt +example_input_file1536.txt +example_input_file1537.txt +example_input_file1538.txt +example_input_file1539.txt +example_input_file1540.txt +example_input_file1541.txt +example_input_file1542.txt +example_input_file1543.txt +example_input_file1544.txt +example_input_file1545.txt +example_input_file1546.txt +example_input_file1547.txt +example_input_file1548.txt +example_input_file1549.txt +example_input_file1550.txt +example_input_file1551.txt +example_input_file1552.txt +example_input_file1553.txt +example_input_file1554.txt +example_input_file1555.txt +example_input_file1556.txt +example_input_file1557.txt +example_input_file1558.txt +example_input_file1559.txt +example_input_file1560.txt +example_input_file1561.txt +example_input_file1562.txt +example_input_file1563.txt +example_input_file1564.txt +example_input_file1565.txt +example_input_file1566.txt +example_input_file1567.txt +example_input_file1568.txt +example_input_file1569.txt +example_input_file1570.txt +example_input_file1571.txt +example_input_file1572.txt +example_input_file1573.txt +example_input_file1574.txt +example_input_file1575.txt +example_input_file1576.txt +example_input_file1577.txt +example_input_file1578.txt +example_input_file1579.txt +example_input_file1580.txt +example_input_file1581.txt +example_input_file1582.txt +example_input_file1583.txt +example_input_file1584.txt +example_input_file1585.txt +example_input_file1586.txt +example_input_file1587.txt +example_input_file1588.txt +example_input_file1589.txt +example_input_file1590.txt +example_input_file1591.txt +example_input_file1592.txt +example_input_file1593.txt +example_input_file1594.txt +example_input_file1595.txt +example_input_file1596.txt +example_input_file1597.txt +example_input_file1598.txt +example_input_file1599.txt +example_input_file1600.txt +example_input_file1601.txt +example_input_file1602.txt +example_input_file1603.txt +example_input_file1604.txt +example_input_file1605.txt +example_input_file1606.txt +example_input_file1607.txt +example_input_file1608.txt +example_input_file1609.txt +example_input_file1610.txt +example_input_file1611.txt +example_input_file1612.txt +example_input_file1613.txt +example_input_file1614.txt +example_input_file1615.txt +example_input_file1616.txt +example_input_file1617.txt +example_input_file1618.txt +example_input_file1619.txt +example_input_file1620.txt +example_input_file1621.txt +example_input_file1622.txt +example_input_file1623.txt +example_input_file1624.txt +example_input_file1625.txt +example_input_file1626.txt +example_input_file1627.txt +example_input_file1628.txt +example_input_file1629.txt +example_input_file1630.txt +example_input_file1631.txt +example_input_file1632.txt +example_input_file1633.txt +example_input_file1634.txt +example_input_file1635.txt +example_input_file1636.txt +example_input_file1637.txt +example_input_file1638.txt +example_input_file1639.txt +example_input_file1640.txt +example_input_file1641.txt +example_input_file1642.txt +example_input_file1643.txt +example_input_file1644.txt +example_input_file1645.txt +example_input_file1646.txt +example_input_file1647.txt +example_input_file1648.txt +example_input_file1649.txt +example_input_file1650.txt +example_input_file1651.txt +example_input_file1652.txt +example_input_file1653.txt +example_input_file1654.txt +example_input_file1655.txt +example_input_file1656.txt +example_input_file1657.txt +example_input_file1658.txt +example_input_file1659.txt +example_input_file1660.txt +example_input_file1661.txt +example_input_file1662.txt +example_input_file1663.txt +example_input_file1664.txt +example_input_file1665.txt +example_input_file1666.txt +example_input_file1667.txt +example_input_file1668.txt +example_input_file1669.txt +example_input_file1670.txt +example_input_file1671.txt +example_input_file1672.txt +example_input_file1673.txt +example_input_file1674.txt +example_input_file1675.txt +example_input_file1676.txt +example_input_file1677.txt +example_input_file1678.txt +example_input_file1679.txt +example_input_file1680.txt +example_input_file1681.txt +example_input_file1682.txt +example_input_file1683.txt +example_input_file1684.txt +example_input_file1685.txt +example_input_file1686.txt +example_input_file1687.txt +example_input_file1688.txt +example_input_file1689.txt +example_input_file1690.txt +example_input_file1691.txt +example_input_file1692.txt +example_input_file1693.txt +example_input_file1694.txt +example_input_file1695.txt +example_input_file1696.txt +example_input_file1697.txt +example_input_file1698.txt +example_input_file1699.txt +example_input_file1700.txt +example_input_file1701.txt +example_input_file1702.txt +example_input_file1703.txt +example_input_file1704.txt +example_input_file1705.txt +example_input_file1706.txt +example_input_file1707.txt +example_input_file1708.txt +example_input_file1709.txt +example_input_file1710.txt +example_input_file1711.txt +example_input_file1712.txt +example_input_file1713.txt +example_input_file1714.txt +example_input_file1715.txt +example_input_file1716.txt +example_input_file1717.txt +example_input_file1718.txt +example_input_file1719.txt +example_input_file1720.txt +example_input_file1721.txt +example_input_file1722.txt +example_input_file1723.txt +example_input_file1724.txt +example_input_file1725.txt +example_input_file1726.txt +example_input_file1727.txt +example_input_file1728.txt +example_input_file1729.txt +example_input_file1730.txt +example_input_file1731.txt +example_input_file1732.txt +example_input_file1733.txt +example_input_file1734.txt +example_input_file1735.txt +example_input_file1736.txt +example_input_file1737.txt +example_input_file1738.txt +example_input_file1739.txt +example_input_file1740.txt +example_input_file1741.txt +example_input_file1742.txt +example_input_file1743.txt +example_input_file1744.txt +example_input_file1745.txt +example_input_file1746.txt +example_input_file1747.txt +example_input_file1748.txt +example_input_file1749.txt +example_input_file1750.txt +example_input_file1751.txt +example_input_file1752.txt +example_input_file1753.txt +example_input_file1754.txt +example_input_file1755.txt +example_input_file1756.txt +example_input_file1757.txt +example_input_file1758.txt +example_input_file1759.txt +example_input_file1760.txt +example_input_file1761.txt +example_input_file1762.txt +example_input_file1763.txt +example_input_file1764.txt +example_input_file1765.txt +example_input_file1766.txt +example_input_file1767.txt +example_input_file1768.txt +example_input_file1769.txt +example_input_file1770.txt +example_input_file1771.txt +example_input_file1772.txt +example_input_file1773.txt +example_input_file1774.txt +example_input_file1775.txt +example_input_file1776.txt +example_input_file1777.txt +example_input_file1778.txt +example_input_file1779.txt +example_input_file1780.txt +example_input_file1781.txt +example_input_file1782.txt +example_input_file1783.txt +example_input_file1784.txt +example_input_file1785.txt +example_input_file1786.txt +example_input_file1787.txt +example_input_file1788.txt +example_input_file1789.txt +example_input_file1790.txt +example_input_file1791.txt +example_input_file1792.txt +example_input_file1793.txt +example_input_file1794.txt +example_input_file1795.txt +example_input_file1796.txt +example_input_file1797.txt +example_input_file1798.txt +example_input_file1799.txt +example_input_file1800.txt +example_input_file1801.txt +example_input_file1802.txt +example_input_file1803.txt +example_input_file1804.txt +example_input_file1805.txt +example_input_file1806.txt +example_input_file1807.txt +example_input_file1808.txt +example_input_file1809.txt +example_input_file1810.txt +example_input_file1811.txt +example_input_file1812.txt +example_input_file1813.txt +example_input_file1814.txt +example_input_file1815.txt +example_input_file1816.txt +example_input_file1817.txt +example_input_file1818.txt +example_input_file1819.txt +example_input_file1820.txt +example_input_file1821.txt +example_input_file1822.txt +example_input_file1823.txt +example_input_file1824.txt +example_input_file1825.txt +example_input_file1826.txt +example_input_file1827.txt +example_input_file1828.txt +example_input_file1829.txt +example_input_file1830.txt +example_input_file1831.txt +example_input_file1832.txt +example_input_file1833.txt +example_input_file1834.txt +example_input_file1835.txt +example_input_file1836.txt +example_input_file1837.txt +example_input_file1838.txt +example_input_file1839.txt +example_input_file1840.txt +example_input_file1841.txt +example_input_file1842.txt +example_input_file1843.txt +example_input_file1844.txt +example_input_file1845.txt +example_input_file1846.txt +example_input_file1847.txt +example_input_file1848.txt +example_input_file1849.txt +example_input_file1850.txt +example_input_file1851.txt +example_input_file1852.txt +example_input_file1853.txt +example_input_file1854.txt +example_input_file1855.txt +example_input_file1856.txt +example_input_file1857.txt +example_input_file1858.txt +example_input_file1859.txt +example_input_file1860.txt +example_input_file1861.txt +example_input_file1862.txt +example_input_file1863.txt +example_input_file1864.txt +example_input_file1865.txt +example_input_file1866.txt +example_input_file1867.txt +example_input_file1868.txt +example_input_file1869.txt +example_input_file1870.txt +example_input_file1871.txt +example_input_file1872.txt +example_input_file1873.txt +example_input_file1874.txt +example_input_file1875.txt +example_input_file1876.txt +example_input_file1877.txt +example_input_file1878.txt +example_input_file1879.txt +example_input_file1880.txt +example_input_file1881.txt +example_input_file1882.txt +example_input_file1883.txt +example_input_file1884.txt +example_input_file1885.txt +example_input_file1886.txt +example_input_file1887.txt +example_input_file1888.txt +example_input_file1889.txt +example_input_file1890.txt +example_input_file1891.txt +example_input_file1892.txt +example_input_file1893.txt +example_input_file1894.txt +example_input_file1895.txt +example_input_file1896.txt +example_input_file1897.txt +example_input_file1898.txt +example_input_file1899.txt +example_input_file1900.txt +example_input_file1901.txt +example_input_file1902.txt +example_input_file1903.txt +example_input_file1904.txt +example_input_file1905.txt +example_input_file1906.txt +example_input_file1907.txt +example_input_file1908.txt +example_input_file1909.txt +example_input_file1910.txt +example_input_file1911.txt +example_input_file1912.txt +example_input_file1913.txt +example_input_file1914.txt +example_input_file1915.txt +example_input_file1916.txt +example_input_file1917.txt +example_input_file1918.txt +example_input_file1919.txt +example_input_file1920.txt +example_input_file1921.txt +example_input_file1922.txt +example_input_file1923.txt +example_input_file1924.txt +example_input_file1925.txt +example_input_file1926.txt +example_input_file1927.txt +example_input_file1928.txt +example_input_file1929.txt +example_input_file1930.txt +example_input_file1931.txt +example_input_file1932.txt +example_input_file1933.txt +example_input_file1934.txt +example_input_file1935.txt +example_input_file1936.txt +example_input_file1937.txt +example_input_file1938.txt +example_input_file1939.txt +example_input_file1940.txt +example_input_file1941.txt +example_input_file1942.txt +example_input_file1943.txt +example_input_file1944.txt +example_input_file1945.txt +example_input_file1946.txt +example_input_file1947.txt +example_input_file1948.txt +example_input_file1949.txt +example_input_file1950.txt +example_input_file1951.txt +example_input_file1952.txt +example_input_file1953.txt +example_input_file1954.txt +example_input_file1955.txt +example_input_file1956.txt +example_input_file1957.txt +example_input_file1958.txt +example_input_file1959.txt +example_input_file1960.txt +example_input_file1961.txt +example_input_file1962.txt +example_input_file1963.txt +example_input_file1964.txt +example_input_file1965.txt +example_input_file1966.txt +example_input_file1967.txt +example_input_file1968.txt +example_input_file1969.txt +example_input_file1970.txt +example_input_file1971.txt +example_input_file1972.txt +example_input_file1973.txt +example_input_file1974.txt +example_input_file1975.txt +example_input_file1976.txt +example_input_file1977.txt +example_input_file1978.txt +example_input_file1979.txt +example_input_file1980.txt +example_input_file1981.txt +example_input_file1982.txt +example_input_file1983.txt +example_input_file1984.txt +example_input_file1985.txt +example_input_file1986.txt +example_input_file1987.txt +example_input_file1988.txt +example_input_file1989.txt +example_input_file1990.txt +example_input_file1991.txt +example_input_file1992.txt +example_input_file1993.txt +example_input_file1994.txt +example_input_file1995.txt +example_input_file1996.txt +example_input_file1997.txt +example_input_file1998.txt +example_input_file1999.txt +example_input_file2000.txt +example_input_file2001.txt +example_input_file2002.txt +example_input_file2003.txt +example_input_file2004.txt +example_input_file2005.txt +example_input_file2006.txt +example_input_file2007.txt +example_input_file2008.txt +example_input_file2009.txt +example_input_file2010.txt +example_input_file2011.txt +example_input_file2012.txt +example_input_file2013.txt +example_input_file2014.txt +example_input_file2015.txt +example_input_file2016.txt +example_input_file2017.txt +example_input_file2018.txt +example_input_file2019.txt +example_input_file2020.txt +example_input_file2021.txt +example_input_file2022.txt +example_input_file2023.txt +example_input_file2024.txt +example_input_file2025.txt +example_input_file2026.txt +example_input_file2027.txt +example_input_file2028.txt +example_input_file2029.txt +example_input_file2030.txt +example_input_file2031.txt +example_input_file2032.txt +example_input_file2033.txt +example_input_file2034.txt +example_input_file2035.txt +example_input_file2036.txt +example_input_file2037.txt +example_input_file2038.txt +example_input_file2039.txt +example_input_file2040.txt +example_input_file2041.txt +example_input_file2042.txt +example_input_file2043.txt +example_input_file2044.txt +example_input_file2045.txt +example_input_file2046.txt +example_input_file2047.txt +example_input_file2048.txt +example_input_file2049.txt +example_input_file2050.txt +example_input_file2051.txt +example_input_file2052.txt +example_input_file2053.txt +example_input_file2054.txt +example_input_file2055.txt +example_input_file2056.txt +example_input_file2057.txt +example_input_file2058.txt +example_input_file2059.txt +example_input_file2060.txt +example_input_file2061.txt +example_input_file2062.txt +example_input_file2063.txt +example_input_file2064.txt +example_input_file2065.txt +example_input_file2066.txt +example_input_file2067.txt +example_input_file2068.txt +example_input_file2069.txt +example_input_file2070.txt +example_input_file2071.txt +example_input_file2072.txt +example_input_file2073.txt +example_input_file2074.txt +example_input_file2075.txt +example_input_file2076.txt +example_input_file2077.txt +example_input_file2078.txt +example_input_file2079.txt +example_input_file2080.txt +example_input_file2081.txt +example_input_file2082.txt +example_input_file2083.txt +example_input_file2084.txt +example_input_file2085.txt +example_input_file2086.txt +example_input_file2087.txt +example_input_file2088.txt +example_input_file2089.txt +example_input_file2090.txt +example_input_file2091.txt +example_input_file2092.txt +example_input_file2093.txt +example_input_file2094.txt +example_input_file2095.txt +example_input_file2096.txt +example_input_file2097.txt +example_input_file2098.txt +example_input_file2099.txt +example_input_file2100.txt +example_input_file2101.txt +example_input_file2102.txt +example_input_file2103.txt +example_input_file2104.txt +example_input_file2105.txt +example_input_file2106.txt +example_input_file2107.txt +example_input_file2108.txt +example_input_file2109.txt +example_input_file2110.txt +example_input_file2111.txt +example_input_file2112.txt +example_input_file2113.txt +example_input_file2114.txt +example_input_file2115.txt +example_input_file2116.txt +example_input_file2117.txt +example_input_file2118.txt +example_input_file2119.txt +example_input_file2120.txt +example_input_file2121.txt +example_input_file2122.txt +example_input_file2123.txt +example_input_file2124.txt +example_input_file2125.txt +example_input_file2126.txt +example_input_file2127.txt +example_input_file2128.txt +example_input_file2129.txt +example_input_file2130.txt +example_input_file2131.txt +example_input_file2132.txt +example_input_file2133.txt +example_input_file2134.txt +example_input_file2135.txt +example_input_file2136.txt +example_input_file2137.txt +example_input_file2138.txt +example_input_file2139.txt +example_input_file2140.txt +example_input_file2141.txt +example_input_file2142.txt +example_input_file2143.txt +example_input_file2144.txt +example_input_file2145.txt +example_input_file2146.txt +example_input_file2147.txt +example_input_file2148.txt +example_input_file2149.txt +example_input_file2150.txt +example_input_file2151.txt +example_input_file2152.txt +example_input_file2153.txt +example_input_file2154.txt +example_input_file2155.txt +example_input_file2156.txt +example_input_file2157.txt +example_input_file2158.txt +example_input_file2159.txt +example_input_file2160.txt +example_input_file2161.txt +example_input_file2162.txt +example_input_file2163.txt +example_input_file2164.txt +example_input_file2165.txt +example_input_file2166.txt +example_input_file2167.txt +example_input_file2168.txt +example_input_file2169.txt +example_input_file2170.txt +example_input_file2171.txt +example_input_file2172.txt +example_input_file2173.txt +example_input_file2174.txt +example_input_file2175.txt +example_input_file2176.txt +example_input_file2177.txt +example_input_file2178.txt +example_input_file2179.txt +example_input_file2180.txt +example_input_file2181.txt +example_input_file2182.txt +example_input_file2183.txt +example_input_file2184.txt +example_input_file2185.txt +example_input_file2186.txt +example_input_file2187.txt +example_input_file2188.txt +example_input_file2189.txt +example_input_file2190.txt +example_input_file2191.txt +example_input_file2192.txt +example_input_file2193.txt +example_input_file2194.txt +example_input_file2195.txt +example_input_file2196.txt +example_input_file2197.txt +example_input_file2198.txt +example_input_file2199.txt +example_input_file2200.txt +example_input_file2201.txt +example_input_file2202.txt +example_input_file2203.txt +example_input_file2204.txt +example_input_file2205.txt +example_input_file2206.txt +example_input_file2207.txt +example_input_file2208.txt +example_input_file2209.txt +example_input_file2210.txt +example_input_file2211.txt +example_input_file2212.txt +example_input_file2213.txt +example_input_file2214.txt +example_input_file2215.txt +example_input_file2216.txt +example_input_file2217.txt +example_input_file2218.txt +example_input_file2219.txt +example_input_file2220.txt +example_input_file2221.txt +example_input_file2222.txt +example_input_file2223.txt +example_input_file2224.txt +example_input_file2225.txt +example_input_file2226.txt +example_input_file2227.txt +example_input_file2228.txt +example_input_file2229.txt +example_input_file2230.txt +example_input_file2231.txt +example_input_file2232.txt +example_input_file2233.txt +example_input_file2234.txt +example_input_file2235.txt +example_input_file2236.txt +example_input_file2237.txt +example_input_file2238.txt +example_input_file2239.txt +example_input_file2240.txt +example_input_file2241.txt +example_input_file2242.txt +example_input_file2243.txt +example_input_file2244.txt +example_input_file2245.txt +example_input_file2246.txt +example_input_file2247.txt +example_input_file2248.txt +example_input_file2249.txt +example_input_file2250.txt +example_input_file2251.txt +example_input_file2252.txt +example_input_file2253.txt +example_input_file2254.txt +example_input_file2255.txt +example_input_file2256.txt +example_input_file2257.txt +example_input_file2258.txt +example_input_file2259.txt +example_input_file2260.txt +example_input_file2261.txt +example_input_file2262.txt +example_input_file2263.txt +example_input_file2264.txt +example_input_file2265.txt +example_input_file2266.txt +example_input_file2267.txt +example_input_file2268.txt +example_input_file2269.txt +example_input_file2270.txt +example_input_file2271.txt +example_input_file2272.txt +example_input_file2273.txt +example_input_file2274.txt +example_input_file2275.txt +example_input_file2276.txt +example_input_file2277.txt +example_input_file2278.txt +example_input_file2279.txt +example_input_file2280.txt +example_input_file2281.txt +example_input_file2282.txt +example_input_file2283.txt +example_input_file2284.txt +example_input_file2285.txt +example_input_file2286.txt +example_input_file2287.txt +example_input_file2288.txt +example_input_file2289.txt +example_input_file2290.txt +example_input_file2291.txt +example_input_file2292.txt +example_input_file2293.txt +example_input_file2294.txt +example_input_file2295.txt +example_input_file2296.txt +example_input_file2297.txt +example_input_file2298.txt +example_input_file2299.txt +example_input_file2300.txt +example_input_file2301.txt +example_input_file2302.txt +example_input_file2303.txt +example_input_file2304.txt +example_input_file2305.txt +example_input_file2306.txt +example_input_file2307.txt +example_input_file2308.txt +example_input_file2309.txt +example_input_file2310.txt +example_input_file2311.txt +example_input_file2312.txt +example_input_file2313.txt +example_input_file2314.txt +example_input_file2315.txt +example_input_file2316.txt +example_input_file2317.txt +example_input_file2318.txt +example_input_file2319.txt +example_input_file2320.txt +example_input_file2321.txt +example_input_file2322.txt +example_input_file2323.txt +example_input_file2324.txt +example_input_file2325.txt +example_input_file2326.txt +example_input_file2327.txt +example_input_file2328.txt +example_input_file2329.txt +example_input_file2330.txt +example_input_file2331.txt +example_input_file2332.txt +example_input_file2333.txt +example_input_file2334.txt +example_input_file2335.txt +example_input_file2336.txt +example_input_file2337.txt +example_input_file2338.txt +example_input_file2339.txt +example_input_file2340.txt +example_input_file2341.txt +example_input_file2342.txt +example_input_file2343.txt +example_input_file2344.txt +example_input_file2345.txt +example_input_file2346.txt +example_input_file2347.txt +example_input_file2348.txt +example_input_file2349.txt +example_input_file2350.txt +example_input_file2351.txt +example_input_file2352.txt +example_input_file2353.txt +example_input_file2354.txt +example_input_file2355.txt +example_input_file2356.txt +example_input_file2357.txt +example_input_file2358.txt +example_input_file2359.txt +example_input_file2360.txt +example_input_file2361.txt +example_input_file2362.txt +example_input_file2363.txt +example_input_file2364.txt +example_input_file2365.txt +example_input_file2366.txt +example_input_file2367.txt +example_input_file2368.txt +example_input_file2369.txt +example_input_file2370.txt +example_input_file2371.txt +example_input_file2372.txt +example_input_file2373.txt +example_input_file2374.txt +example_input_file2375.txt +example_input_file2376.txt +example_input_file2377.txt +example_input_file2378.txt +example_input_file2379.txt +example_input_file2380.txt +example_input_file2381.txt +example_input_file2382.txt +example_input_file2383.txt +example_input_file2384.txt +example_input_file2385.txt +example_input_file2386.txt +example_input_file2387.txt +example_input_file2388.txt +example_input_file2389.txt +example_input_file2390.txt +example_input_file2391.txt +example_input_file2392.txt +example_input_file2393.txt +example_input_file2394.txt +example_input_file2395.txt +example_input_file2396.txt +example_input_file2397.txt +example_input_file2398.txt +example_input_file2399.txt +example_input_file2400.txt +example_input_file2401.txt +example_input_file2402.txt +example_input_file2403.txt +example_input_file2404.txt +example_input_file2405.txt +example_input_file2406.txt +example_input_file2407.txt +example_input_file2408.txt +example_input_file2409.txt +example_input_file2410.txt +example_input_file2411.txt +example_input_file2412.txt +example_input_file2413.txt +example_input_file2414.txt +example_input_file2415.txt +example_input_file2416.txt +example_input_file2417.txt +example_input_file2418.txt +example_input_file2419.txt +example_input_file2420.txt +example_input_file2421.txt +example_input_file2422.txt +example_input_file2423.txt +example_input_file2424.txt +example_input_file2425.txt +example_input_file2426.txt +example_input_file2427.txt +example_input_file2428.txt +example_input_file2429.txt +example_input_file2430.txt +example_input_file2431.txt +example_input_file2432.txt +example_input_file2433.txt +example_input_file2434.txt +example_input_file2435.txt +example_input_file2436.txt +example_input_file2437.txt +example_input_file2438.txt +example_input_file2439.txt +example_input_file2440.txt +example_input_file2441.txt +example_input_file2442.txt +example_input_file2443.txt +example_input_file2444.txt +example_input_file2445.txt +example_input_file2446.txt +example_input_file2447.txt +example_input_file2448.txt +example_input_file2449.txt +example_input_file2450.txt +example_input_file2451.txt +example_input_file2452.txt +example_input_file2453.txt +example_input_file2454.txt +example_input_file2455.txt +example_input_file2456.txt +example_input_file2457.txt +example_input_file2458.txt +example_input_file2459.txt +example_input_file2460.txt +example_input_file2461.txt +example_input_file2462.txt +example_input_file2463.txt +example_input_file2464.txt +example_input_file2465.txt +example_input_file2466.txt +example_input_file2467.txt +example_input_file2468.txt +example_input_file2469.txt +example_input_file2470.txt +example_input_file2471.txt +example_input_file2472.txt +example_input_file2473.txt +example_input_file2474.txt +example_input_file2475.txt +example_input_file2476.txt +example_input_file2477.txt +example_input_file2478.txt +example_input_file2479.txt +example_input_file2480.txt +example_input_file2481.txt +example_input_file2482.txt +example_input_file2483.txt +example_input_file2484.txt +example_input_file2485.txt +example_input_file2486.txt +example_input_file2487.txt +example_input_file2488.txt +example_input_file2489.txt +example_input_file2490.txt +example_input_file2491.txt +example_input_file2492.txt +example_input_file2493.txt +example_input_file2494.txt +example_input_file2495.txt +example_input_file2496.txt +example_input_file2497.txt +example_input_file2498.txt +example_input_file2499.txt +example_input_file2500.txt +example_input_file2501.txt +example_input_file2502.txt +example_input_file2503.txt +example_input_file2504.txt +example_input_file2505.txt +example_input_file2506.txt +example_input_file2507.txt +example_input_file2508.txt +example_input_file2509.txt +example_input_file2510.txt +example_input_file2511.txt +example_input_file2512.txt +example_input_file2513.txt +example_input_file2514.txt +example_input_file2515.txt +example_input_file2516.txt +example_input_file2517.txt +example_input_file2518.txt +example_input_file2519.txt +example_input_file2520.txt +example_input_file2521.txt +example_input_file2522.txt +example_input_file2523.txt +example_input_file2524.txt +example_input_file2525.txt +example_input_file2526.txt +example_input_file2527.txt +example_input_file2528.txt +example_input_file2529.txt +example_input_file2530.txt +example_input_file2531.txt +example_input_file2532.txt +example_input_file2533.txt +example_input_file2534.txt +example_input_file2535.txt +example_input_file2536.txt +example_input_file2537.txt +example_input_file2538.txt +example_input_file2539.txt +example_input_file2540.txt +example_input_file2541.txt +example_input_file2542.txt +example_input_file2543.txt +example_input_file2544.txt +example_input_file2545.txt +example_input_file2546.txt +example_input_file2547.txt +example_input_file2548.txt +example_input_file2549.txt +example_input_file2550.txt +example_input_file2551.txt +example_input_file2552.txt +example_input_file2553.txt +example_input_file2554.txt +example_input_file2555.txt +example_input_file2556.txt +example_input_file2557.txt +example_input_file2558.txt +example_input_file2559.txt +example_input_file2560.txt +example_input_file2561.txt +example_input_file2562.txt +example_input_file2563.txt +example_input_file2564.txt +example_input_file2565.txt +example_input_file2566.txt +example_input_file2567.txt +example_input_file2568.txt +example_input_file2569.txt +example_input_file2570.txt +example_input_file2571.txt +example_input_file2572.txt +example_input_file2573.txt +example_input_file2574.txt +example_input_file2575.txt +example_input_file2576.txt +example_input_file2577.txt +example_input_file2578.txt +example_input_file2579.txt +example_input_file2580.txt +example_input_file2581.txt +example_input_file2582.txt +example_input_file2583.txt +example_input_file2584.txt +example_input_file2585.txt +example_input_file2586.txt +example_input_file2587.txt +example_input_file2588.txt +example_input_file2589.txt +example_input_file2590.txt +example_input_file2591.txt +example_input_file2592.txt +example_input_file2593.txt +example_input_file2594.txt +example_input_file2595.txt +example_input_file2596.txt +example_input_file2597.txt +example_input_file2598.txt +example_input_file2599.txt +example_input_file2600.txt +example_input_file2601.txt +example_input_file2602.txt +example_input_file2603.txt +example_input_file2604.txt +example_input_file2605.txt +example_input_file2606.txt +example_input_file2607.txt +example_input_file2608.txt +example_input_file2609.txt +example_input_file2610.txt +example_input_file2611.txt +example_input_file2612.txt +example_input_file2613.txt +example_input_file2614.txt +example_input_file2615.txt +example_input_file2616.txt +example_input_file2617.txt +example_input_file2618.txt +example_input_file2619.txt +example_input_file2620.txt +example_input_file2621.txt +example_input_file2622.txt +example_input_file2623.txt +example_input_file2624.txt +example_input_file2625.txt +example_input_file2626.txt +example_input_file2627.txt +example_input_file2628.txt +example_input_file2629.txt +example_input_file2630.txt +example_input_file2631.txt +example_input_file2632.txt +example_input_file2633.txt +example_input_file2634.txt +example_input_file2635.txt +example_input_file2636.txt +example_input_file2637.txt +example_input_file2638.txt +example_input_file2639.txt +example_input_file2640.txt +example_input_file2641.txt +example_input_file2642.txt +example_input_file2643.txt +example_input_file2644.txt +example_input_file2645.txt +example_input_file2646.txt +example_input_file2647.txt +example_input_file2648.txt +example_input_file2649.txt +example_input_file2650.txt +example_input_file2651.txt +example_input_file2652.txt +example_input_file2653.txt +example_input_file2654.txt +example_input_file2655.txt +example_input_file2656.txt +example_input_file2657.txt +example_input_file2658.txt +example_input_file2659.txt +example_input_file2660.txt +example_input_file2661.txt +example_input_file2662.txt +example_input_file2663.txt +example_input_file2664.txt +example_input_file2665.txt +example_input_file2666.txt +example_input_file2667.txt +example_input_file2668.txt +example_input_file2669.txt +example_input_file2670.txt +example_input_file2671.txt +example_input_file2672.txt +example_input_file2673.txt +example_input_file2674.txt +example_input_file2675.txt +example_input_file2676.txt +example_input_file2677.txt +example_input_file2678.txt +example_input_file2679.txt +example_input_file2680.txt +example_input_file2681.txt +example_input_file2682.txt +example_input_file2683.txt +example_input_file2684.txt +example_input_file2685.txt +example_input_file2686.txt +example_input_file2687.txt +example_input_file2688.txt +example_input_file2689.txt +example_input_file2690.txt +example_input_file2691.txt +example_input_file2692.txt +example_input_file2693.txt +example_input_file2694.txt +example_input_file2695.txt +example_input_file2696.txt +example_input_file2697.txt +example_input_file2698.txt +example_input_file2699.txt +example_input_file2700.txt +example_input_file2701.txt +example_input_file2702.txt +example_input_file2703.txt +example_input_file2704.txt +example_input_file2705.txt +example_input_file2706.txt +example_input_file2707.txt +example_input_file2708.txt +example_input_file2709.txt +example_input_file2710.txt +example_input_file2711.txt +example_input_file2712.txt +example_input_file2713.txt +example_input_file2714.txt +example_input_file2715.txt +example_input_file2716.txt +example_input_file2717.txt +example_input_file2718.txt +example_input_file2719.txt +example_input_file2720.txt +example_input_file2721.txt +example_input_file2722.txt +example_input_file2723.txt +example_input_file2724.txt +example_input_file2725.txt +example_input_file2726.txt +example_input_file2727.txt +example_input_file2728.txt +example_input_file2729.txt +example_input_file2730.txt +example_input_file2731.txt +example_input_file2732.txt +example_input_file2733.txt +example_input_file2734.txt +example_input_file2735.txt +example_input_file2736.txt +example_input_file2737.txt +example_input_file2738.txt +example_input_file2739.txt +example_input_file2740.txt +example_input_file2741.txt +example_input_file2742.txt +example_input_file2743.txt +example_input_file2744.txt +example_input_file2745.txt +example_input_file2746.txt +example_input_file2747.txt +example_input_file2748.txt +example_input_file2749.txt +example_input_file2750.txt +example_input_file2751.txt +example_input_file2752.txt +example_input_file2753.txt +example_input_file2754.txt +example_input_file2755.txt +example_input_file2756.txt +example_input_file2757.txt +example_input_file2758.txt +example_input_file2759.txt +example_input_file2760.txt +example_input_file2761.txt +example_input_file2762.txt +example_input_file2763.txt +example_input_file2764.txt +example_input_file2765.txt +example_input_file2766.txt +example_input_file2767.txt +example_input_file2768.txt +example_input_file2769.txt +example_input_file2770.txt +example_input_file2771.txt +example_input_file2772.txt +example_input_file2773.txt +example_input_file2774.txt +example_input_file2775.txt +example_input_file2776.txt +example_input_file2777.txt +example_input_file2778.txt +example_input_file2779.txt +example_input_file2780.txt +example_input_file2781.txt +example_input_file2782.txt +example_input_file2783.txt +example_input_file2784.txt +example_input_file2785.txt +example_input_file2786.txt +example_input_file2787.txt +example_input_file2788.txt +example_input_file2789.txt +example_input_file2790.txt +example_input_file2791.txt +example_input_file2792.txt +example_input_file2793.txt +example_input_file2794.txt +example_input_file2795.txt +example_input_file2796.txt +example_input_file2797.txt +example_input_file2798.txt +example_input_file2799.txt +example_input_file2800.txt +example_input_file2801.txt +example_input_file2802.txt +example_input_file2803.txt +example_input_file2804.txt +example_input_file2805.txt +example_input_file2806.txt +example_input_file2807.txt +example_input_file2808.txt +example_input_file2809.txt +example_input_file2810.txt +example_input_file2811.txt +example_input_file2812.txt +example_input_file2813.txt +example_input_file2814.txt +example_input_file2815.txt +example_input_file2816.txt +example_input_file2817.txt +example_input_file2818.txt +example_input_file2819.txt +example_input_file2820.txt +example_input_file2821.txt +example_input_file2822.txt +example_input_file2823.txt +example_input_file2824.txt +example_input_file2825.txt +example_input_file2826.txt +example_input_file2827.txt +example_input_file2828.txt +example_input_file2829.txt +example_input_file2830.txt +example_input_file2831.txt +example_input_file2832.txt +example_input_file2833.txt +example_input_file2834.txt +example_input_file2835.txt +example_input_file2836.txt +example_input_file2837.txt +example_input_file2838.txt +example_input_file2839.txt +example_input_file2840.txt +example_input_file2841.txt +example_input_file2842.txt +example_input_file2843.txt +example_input_file2844.txt +example_input_file2845.txt +example_input_file2846.txt +example_input_file2847.txt +example_input_file2848.txt +example_input_file2849.txt +example_input_file2850.txt +example_input_file2851.txt +example_input_file2852.txt +example_input_file2853.txt +example_input_file2854.txt +example_input_file2855.txt +example_input_file2856.txt +example_input_file2857.txt +example_input_file2858.txt +example_input_file2859.txt +example_input_file2860.txt +example_input_file2861.txt +example_input_file2862.txt +example_input_file2863.txt +example_input_file2864.txt +example_input_file2865.txt +example_input_file2866.txt +example_input_file2867.txt +example_input_file2868.txt +example_input_file2869.txt +example_input_file2870.txt +example_input_file2871.txt +example_input_file2872.txt +example_input_file2873.txt +example_input_file2874.txt +example_input_file2875.txt +example_input_file2876.txt +example_input_file2877.txt +example_input_file2878.txt +example_input_file2879.txt +example_input_file2880.txt +example_input_file2881.txt +example_input_file2882.txt +example_input_file2883.txt +example_input_file2884.txt +example_input_file2885.txt +example_input_file2886.txt +example_input_file2887.txt +example_input_file2888.txt +example_input_file2889.txt +example_input_file2890.txt +example_input_file2891.txt +example_input_file2892.txt +example_input_file2893.txt +example_input_file2894.txt +example_input_file2895.txt +example_input_file2896.txt +example_input_file2897.txt +example_input_file2898.txt +example_input_file2899.txt +example_input_file2900.txt +example_input_file2901.txt +example_input_file2902.txt +example_input_file2903.txt +example_input_file2904.txt +example_input_file2905.txt +example_input_file2906.txt +example_input_file2907.txt +example_input_file2908.txt +example_input_file2909.txt +example_input_file2910.txt +example_input_file2911.txt +example_input_file2912.txt +example_input_file2913.txt +example_input_file2914.txt +example_input_file2915.txt +example_input_file2916.txt +example_input_file2917.txt +example_input_file2918.txt +example_input_file2919.txt +example_input_file2920.txt +example_input_file2921.txt +example_input_file2922.txt +example_input_file2923.txt +example_input_file2924.txt +example_input_file2925.txt +example_input_file2926.txt +example_input_file2927.txt +example_input_file2928.txt +example_input_file2929.txt +example_input_file2930.txt +example_input_file2931.txt +example_input_file2932.txt +example_input_file2933.txt +example_input_file2934.txt +example_input_file2935.txt +example_input_file2936.txt +example_input_file2937.txt +example_input_file2938.txt +example_input_file2939.txt +example_input_file2940.txt +example_input_file2941.txt +example_input_file2942.txt +example_input_file2943.txt +example_input_file2944.txt +example_input_file2945.txt +example_input_file2946.txt +example_input_file2947.txt +example_input_file2948.txt +example_input_file2949.txt +example_input_file2950.txt +example_input_file2951.txt +example_input_file2952.txt +example_input_file2953.txt +example_input_file2954.txt +example_input_file2955.txt +example_input_file2956.txt +example_input_file2957.txt +example_input_file2958.txt +example_input_file2959.txt +example_input_file2960.txt +example_input_file2961.txt +example_input_file2962.txt +example_input_file2963.txt +example_input_file2964.txt +example_input_file2965.txt +example_input_file2966.txt +example_input_file2967.txt +example_input_file2968.txt +example_input_file2969.txt +example_input_file2970.txt +example_input_file2971.txt +example_input_file2972.txt +example_input_file2973.txt +example_input_file2974.txt +example_input_file2975.txt +example_input_file2976.txt +example_input_file2977.txt +example_input_file2978.txt +example_input_file2979.txt +example_input_file2980.txt +example_input_file2981.txt +example_input_file2982.txt +example_input_file2983.txt +example_input_file2984.txt +example_input_file2985.txt +example_input_file2986.txt +example_input_file2987.txt +example_input_file2988.txt +example_input_file2989.txt +example_input_file2990.txt +example_input_file2991.txt +example_input_file2992.txt +example_input_file2993.txt +example_input_file2994.txt +example_input_file2995.txt +example_input_file2996.txt +example_input_file2997.txt +example_input_file2998.txt +example_input_file2999.txt +example_input_file3000.txt +example_input_file3001.txt +example_input_file3002.txt +example_input_file3003.txt +example_input_file3004.txt +example_input_file3005.txt +example_input_file3006.txt +example_input_file3007.txt +example_input_file3008.txt +example_input_file3009.txt +example_input_file3010.txt +example_input_file3011.txt +example_input_file3012.txt +example_input_file3013.txt +example_input_file3014.txt +example_input_file3015.txt +example_input_file3016.txt +example_input_file3017.txt +example_input_file3018.txt +example_input_file3019.txt +example_input_file3020.txt +example_input_file3021.txt +example_input_file3022.txt +example_input_file3023.txt +example_input_file3024.txt +example_input_file3025.txt +example_input_file3026.txt +example_input_file3027.txt +example_input_file3028.txt +example_input_file3029.txt +example_input_file3030.txt +example_input_file3031.txt +example_input_file3032.txt +example_input_file3033.txt +example_input_file3034.txt +example_input_file3035.txt +example_input_file3036.txt +example_input_file3037.txt +example_input_file3038.txt +example_input_file3039.txt +example_input_file3040.txt +example_input_file3041.txt +example_input_file3042.txt +example_input_file3043.txt +example_input_file3044.txt +example_input_file3045.txt +example_input_file3046.txt +example_input_file3047.txt +example_input_file3048.txt +example_input_file3049.txt +example_input_file3050.txt +example_input_file3051.txt +example_input_file3052.txt +example_input_file3053.txt +example_input_file3054.txt +example_input_file3055.txt +example_input_file3056.txt +example_input_file3057.txt +example_input_file3058.txt +example_input_file3059.txt +example_input_file3060.txt +example_input_file3061.txt +example_input_file3062.txt +example_input_file3063.txt +example_input_file3064.txt +example_input_file3065.txt +example_input_file3066.txt +example_input_file3067.txt +example_input_file3068.txt +example_input_file3069.txt +example_input_file3070.txt +example_input_file3071.txt +example_input_file3072.txt +example_input_file3073.txt +example_input_file3074.txt +example_input_file3075.txt +example_input_file3076.txt +example_input_file3077.txt +example_input_file3078.txt +example_input_file3079.txt +example_input_file3080.txt +example_input_file3081.txt +example_input_file3082.txt +example_input_file3083.txt +example_input_file3084.txt +example_input_file3085.txt +example_input_file3086.txt +example_input_file3087.txt +example_input_file3088.txt +example_input_file3089.txt +example_input_file3090.txt +example_input_file3091.txt +example_input_file3092.txt +example_input_file3093.txt +example_input_file3094.txt +example_input_file3095.txt +example_input_file3096.txt +example_input_file3097.txt +example_input_file3098.txt +example_input_file3099.txt +example_input_file3100.txt +example_input_file3101.txt +example_input_file3102.txt +example_input_file3103.txt +example_input_file3104.txt +example_input_file3105.txt +example_input_file3106.txt +example_input_file3107.txt +example_input_file3108.txt +example_input_file3109.txt +example_input_file3110.txt +example_input_file3111.txt +example_input_file3112.txt +example_input_file3113.txt +example_input_file3114.txt +example_input_file3115.txt +example_input_file3116.txt +example_input_file3117.txt +example_input_file3118.txt +example_input_file3119.txt +example_input_file3120.txt +example_input_file3121.txt +example_input_file3122.txt +example_input_file3123.txt +example_input_file3124.txt +example_input_file3125.txt +example_input_file3126.txt +example_input_file3127.txt +example_input_file3128.txt +example_input_file3129.txt +example_input_file3130.txt +example_input_file3131.txt +example_input_file3132.txt +example_input_file3133.txt +example_input_file3134.txt +example_input_file3135.txt +example_input_file3136.txt +example_input_file3137.txt +example_input_file3138.txt +example_input_file3139.txt +example_input_file3140.txt +example_input_file3141.txt +example_input_file3142.txt +example_input_file3143.txt +example_input_file3144.txt +example_input_file3145.txt +example_input_file3146.txt +example_input_file3147.txt +example_input_file3148.txt +example_input_file3149.txt +example_input_file3150.txt +example_input_file3151.txt +example_input_file3152.txt +example_input_file3153.txt +example_input_file3154.txt +example_input_file3155.txt +example_input_file3156.txt +example_input_file3157.txt +example_input_file3158.txt +example_input_file3159.txt +example_input_file3160.txt +example_input_file3161.txt +example_input_file3162.txt +example_input_file3163.txt +example_input_file3164.txt +example_input_file3165.txt +example_input_file3166.txt +example_input_file3167.txt +example_input_file3168.txt +example_input_file3169.txt +example_input_file3170.txt +example_input_file3171.txt +example_input_file3172.txt +example_input_file3173.txt +example_input_file3174.txt +example_input_file3175.txt +example_input_file3176.txt +example_input_file3177.txt +example_input_file3178.txt +example_input_file3179.txt +example_input_file3180.txt +example_input_file3181.txt +example_input_file3182.txt +example_input_file3183.txt +example_input_file3184.txt +example_input_file3185.txt +example_input_file3186.txt +example_input_file3187.txt +example_input_file3188.txt +example_input_file3189.txt +example_input_file3190.txt +example_input_file3191.txt +example_input_file3192.txt +example_input_file3193.txt +example_input_file3194.txt +example_input_file3195.txt +example_input_file3196.txt +example_input_file3197.txt +example_input_file3198.txt +example_input_file3199.txt +example_input_file3200.txt +example_input_file3201.txt +example_input_file3202.txt +example_input_file3203.txt +example_input_file3204.txt +example_input_file3205.txt +example_input_file3206.txt +example_input_file3207.txt +example_input_file3208.txt +example_input_file3209.txt +example_input_file3210.txt +example_input_file3211.txt +example_input_file3212.txt +example_input_file3213.txt +example_input_file3214.txt +example_input_file3215.txt +example_input_file3216.txt +example_input_file3217.txt +example_input_file3218.txt +example_input_file3219.txt +example_input_file3220.txt +example_input_file3221.txt +example_input_file3222.txt +example_input_file3223.txt +example_input_file3224.txt +example_input_file3225.txt +example_input_file3226.txt +example_input_file3227.txt +example_input_file3228.txt +example_input_file3229.txt +example_input_file3230.txt +example_input_file3231.txt +example_input_file3232.txt +example_input_file3233.txt +example_input_file3234.txt +example_input_file3235.txt +example_input_file3236.txt +example_input_file3237.txt +example_input_file3238.txt +example_input_file3239.txt +example_input_file3240.txt +example_input_file3241.txt +example_input_file3242.txt +example_input_file3243.txt +example_input_file3244.txt +example_input_file3245.txt +example_input_file3246.txt +example_input_file3247.txt +example_input_file3248.txt +example_input_file3249.txt +example_input_file3250.txt +example_input_file3251.txt +example_input_file3252.txt +example_input_file3253.txt +example_input_file3254.txt +example_input_file3255.txt +example_input_file3256.txt +example_input_file3257.txt +example_input_file3258.txt +example_input_file3259.txt +example_input_file3260.txt +example_input_file3261.txt +example_input_file3262.txt +example_input_file3263.txt +example_input_file3264.txt +example_input_file3265.txt +example_input_file3266.txt +example_input_file3267.txt +example_input_file3268.txt +example_input_file3269.txt +example_input_file3270.txt +example_input_file3271.txt +example_input_file3272.txt +example_input_file3273.txt +example_input_file3274.txt +example_input_file3275.txt +example_input_file3276.txt +example_input_file3277.txt +example_input_file3278.txt +example_input_file3279.txt +example_input_file3280.txt +example_input_file3281.txt +example_input_file3282.txt +example_input_file3283.txt +example_input_file3284.txt +example_input_file3285.txt +example_input_file3286.txt +example_input_file3287.txt +example_input_file3288.txt +example_input_file3289.txt +example_input_file3290.txt +example_input_file3291.txt +example_input_file3292.txt +example_input_file3293.txt +example_input_file3294.txt +example_input_file3295.txt +example_input_file3296.txt +example_input_file3297.txt +example_input_file3298.txt +example_input_file3299.txt +example_input_file3300.txt +example_input_file3301.txt +example_input_file3302.txt +example_input_file3303.txt +example_input_file3304.txt +example_input_file3305.txt +example_input_file3306.txt +example_input_file3307.txt +example_input_file3308.txt +example_input_file3309.txt +example_input_file3310.txt +example_input_file3311.txt +example_input_file3312.txt +example_input_file3313.txt +example_input_file3314.txt +example_input_file3315.txt +example_input_file3316.txt +example_input_file3317.txt +example_input_file3318.txt +example_input_file3319.txt +example_input_file3320.txt +example_input_file3321.txt +example_input_file3322.txt +example_input_file3323.txt +example_input_file3324.txt +example_input_file3325.txt +example_input_file3326.txt +example_input_file3327.txt +example_input_file3328.txt +example_input_file3329.txt +example_input_file3330.txt +example_input_file3331.txt +example_input_file3332.txt +example_input_file3333.txt +example_input_file3334.txt +example_input_file3335.txt +example_input_file3336.txt +example_input_file3337.txt +example_input_file3338.txt +example_input_file3339.txt +example_input_file3340.txt +example_input_file3341.txt +example_input_file3342.txt +example_input_file3343.txt +example_input_file3344.txt +example_input_file3345.txt +example_input_file3346.txt +example_input_file3347.txt +example_input_file3348.txt +example_input_file3349.txt +example_input_file3350.txt +example_input_file3351.txt +example_input_file3352.txt +example_input_file3353.txt +example_input_file3354.txt +example_input_file3355.txt +example_input_file3356.txt +example_input_file3357.txt +example_input_file3358.txt +example_input_file3359.txt +example_input_file3360.txt +example_input_file3361.txt +example_input_file3362.txt +example_input_file3363.txt +example_input_file3364.txt +example_input_file3365.txt +example_input_file3366.txt +example_input_file3367.txt +example_input_file3368.txt +example_input_file3369.txt +example_input_file3370.txt +example_input_file3371.txt +example_input_file3372.txt +example_input_file3373.txt +example_input_file3374.txt +example_input_file3375.txt +example_input_file3376.txt +example_input_file3377.txt +example_input_file3378.txt +example_input_file3379.txt +example_input_file3380.txt +example_input_file3381.txt +example_input_file3382.txt +example_input_file3383.txt +example_input_file3384.txt +example_input_file3385.txt +example_input_file3386.txt +example_input_file3387.txt +example_input_file3388.txt +example_input_file3389.txt +example_input_file3390.txt +example_input_file3391.txt +example_input_file3392.txt +example_input_file3393.txt +example_input_file3394.txt +example_input_file3395.txt +example_input_file3396.txt +example_input_file3397.txt +example_input_file3398.txt +example_input_file3399.txt +example_input_file3400.txt +example_input_file3401.txt +example_input_file3402.txt +example_input_file3403.txt +example_input_file3404.txt +example_input_file3405.txt +example_input_file3406.txt +example_input_file3407.txt +example_input_file3408.txt +example_input_file3409.txt +example_input_file3410.txt +example_input_file3411.txt +example_input_file3412.txt +example_input_file3413.txt +example_input_file3414.txt +example_input_file3415.txt +example_input_file3416.txt +example_input_file3417.txt +example_input_file3418.txt +example_input_file3419.txt +example_input_file3420.txt +example_input_file3421.txt +example_input_file3422.txt +example_input_file3423.txt +example_input_file3424.txt +example_input_file3425.txt +example_input_file3426.txt +example_input_file3427.txt +example_input_file3428.txt +example_input_file3429.txt +example_input_file3430.txt +example_input_file3431.txt +example_input_file3432.txt +example_input_file3433.txt +example_input_file3434.txt +example_input_file3435.txt +example_input_file3436.txt +example_input_file3437.txt +example_input_file3438.txt +example_input_file3439.txt +example_input_file3440.txt +example_input_file3441.txt +example_input_file3442.txt +example_input_file3443.txt +example_input_file3444.txt +example_input_file3445.txt +example_input_file3446.txt +example_input_file3447.txt +example_input_file3448.txt +example_input_file3449.txt +example_input_file3450.txt +example_input_file3451.txt +example_input_file3452.txt +example_input_file3453.txt +example_input_file3454.txt +example_input_file3455.txt +example_input_file3456.txt +example_input_file3457.txt +example_input_file3458.txt +example_input_file3459.txt +example_input_file3460.txt +example_input_file3461.txt +example_input_file3462.txt +example_input_file3463.txt +example_input_file3464.txt +example_input_file3465.txt +example_input_file3466.txt +example_input_file3467.txt +example_input_file3468.txt +example_input_file3469.txt +example_input_file3470.txt +example_input_file3471.txt +example_input_file3472.txt +example_input_file3473.txt +example_input_file3474.txt +example_input_file3475.txt +example_input_file3476.txt +example_input_file3477.txt +example_input_file3478.txt +example_input_file3479.txt +example_input_file3480.txt +example_input_file3481.txt +example_input_file3482.txt +example_input_file3483.txt +example_input_file3484.txt +example_input_file3485.txt +example_input_file3486.txt +example_input_file3487.txt +example_input_file3488.txt +example_input_file3489.txt +example_input_file3490.txt +example_input_file3491.txt +example_input_file3492.txt +example_input_file3493.txt +example_input_file3494.txt +example_input_file3495.txt +example_input_file3496.txt +example_input_file3497.txt +example_input_file3498.txt +example_input_file3499.txt +example_input_file3500.txt +example_input_file3501.txt +example_input_file3502.txt +example_input_file3503.txt +example_input_file3504.txt +example_input_file3505.txt +example_input_file3506.txt +example_input_file3507.txt +example_input_file3508.txt +example_input_file3509.txt +example_input_file3510.txt +example_input_file3511.txt +example_input_file3512.txt +example_input_file3513.txt +example_input_file3514.txt +example_input_file3515.txt +example_input_file3516.txt +example_input_file3517.txt +example_input_file3518.txt +example_input_file3519.txt +example_input_file3520.txt +example_input_file3521.txt +example_input_file3522.txt +example_input_file3523.txt +example_input_file3524.txt +example_input_file3525.txt +example_input_file3526.txt +example_input_file3527.txt +example_input_file3528.txt +example_input_file3529.txt +example_input_file3530.txt +example_input_file3531.txt +example_input_file3532.txt +example_input_file3533.txt +example_input_file3534.txt +example_input_file3535.txt +example_input_file3536.txt +example_input_file3537.txt +example_input_file3538.txt +example_input_file3539.txt +example_input_file3540.txt +example_input_file3541.txt +example_input_file3542.txt +example_input_file3543.txt +example_input_file3544.txt +example_input_file3545.txt +example_input_file3546.txt +example_input_file3547.txt +example_input_file3548.txt +example_input_file3549.txt +example_input_file3550.txt +example_input_file3551.txt +example_input_file3552.txt +example_input_file3553.txt +example_input_file3554.txt +example_input_file3555.txt +example_input_file3556.txt +example_input_file3557.txt +example_input_file3558.txt +example_input_file3559.txt +example_input_file3560.txt +example_input_file3561.txt +example_input_file3562.txt +example_input_file3563.txt +example_input_file3564.txt +example_input_file3565.txt +example_input_file3566.txt +example_input_file3567.txt +example_input_file3568.txt +example_input_file3569.txt +example_input_file3570.txt +example_input_file3571.txt +example_input_file3572.txt +example_input_file3573.txt +example_input_file3574.txt +example_input_file3575.txt +example_input_file3576.txt +example_input_file3577.txt +example_input_file3578.txt +example_input_file3579.txt +example_input_file3580.txt +example_input_file3581.txt +example_input_file3582.txt +example_input_file3583.txt +example_input_file3584.txt +example_input_file3585.txt +example_input_file3586.txt +example_input_file3587.txt +example_input_file3588.txt +example_input_file3589.txt +example_input_file3590.txt +example_input_file3591.txt +example_input_file3592.txt +example_input_file3593.txt +example_input_file3594.txt +example_input_file3595.txt +example_input_file3596.txt +example_input_file3597.txt +example_input_file3598.txt +example_input_file3599.txt +example_input_file3600.txt +example_input_file3601.txt +example_input_file3602.txt +example_input_file3603.txt +example_input_file3604.txt +example_input_file3605.txt +example_input_file3606.txt +example_input_file3607.txt +example_input_file3608.txt +example_input_file3609.txt +example_input_file3610.txt +example_input_file3611.txt +example_input_file3612.txt +example_input_file3613.txt +example_input_file3614.txt +example_input_file3615.txt +example_input_file3616.txt +example_input_file3617.txt +example_input_file3618.txt +example_input_file3619.txt +example_input_file3620.txt +example_input_file3621.txt +example_input_file3622.txt +example_input_file3623.txt +example_input_file3624.txt +example_input_file3625.txt +example_input_file3626.txt +example_input_file3627.txt +example_input_file3628.txt +example_input_file3629.txt +example_input_file3630.txt +example_input_file3631.txt +example_input_file3632.txt +example_input_file3633.txt +example_input_file3634.txt +example_input_file3635.txt +example_input_file3636.txt +example_input_file3637.txt +example_input_file3638.txt +example_input_file3639.txt +example_input_file3640.txt +example_input_file3641.txt +example_input_file3642.txt +example_input_file3643.txt +example_input_file3644.txt +example_input_file3645.txt +example_input_file3646.txt +example_input_file3647.txt +example_input_file3648.txt +example_input_file3649.txt +example_input_file3650.txt +example_input_file3651.txt +example_input_file3652.txt +example_input_file3653.txt +example_input_file3654.txt +example_input_file3655.txt +example_input_file3656.txt +example_input_file3657.txt +example_input_file3658.txt +example_input_file3659.txt +example_input_file3660.txt +example_input_file3661.txt +example_input_file3662.txt +example_input_file3663.txt +example_input_file3664.txt +example_input_file3665.txt +example_input_file3666.txt +example_input_file3667.txt +example_input_file3668.txt +example_input_file3669.txt +example_input_file3670.txt +example_input_file3671.txt +example_input_file3672.txt +example_input_file3673.txt +example_input_file3674.txt +example_input_file3675.txt +example_input_file3676.txt +example_input_file3677.txt +example_input_file3678.txt +example_input_file3679.txt +example_input_file3680.txt +example_input_file3681.txt +example_input_file3682.txt +example_input_file3683.txt +example_input_file3684.txt +example_input_file3685.txt +example_input_file3686.txt +example_input_file3687.txt +example_input_file3688.txt +example_input_file3689.txt +example_input_file3690.txt +example_input_file3691.txt +example_input_file3692.txt +example_input_file3693.txt +example_input_file3694.txt +example_input_file3695.txt +example_input_file3696.txt +example_input_file3697.txt +example_input_file3698.txt +example_input_file3699.txt +example_input_file3700.txt +example_input_file3701.txt +example_input_file3702.txt +example_input_file3703.txt +example_input_file3704.txt +example_input_file3705.txt +example_input_file3706.txt +example_input_file3707.txt +example_input_file3708.txt +example_input_file3709.txt +example_input_file3710.txt +example_input_file3711.txt +example_input_file3712.txt +example_input_file3713.txt +example_input_file3714.txt +example_input_file3715.txt +example_input_file3716.txt +example_input_file3717.txt +example_input_file3718.txt +example_input_file3719.txt +example_input_file3720.txt +example_input_file3721.txt +example_input_file3722.txt +example_input_file3723.txt +example_input_file3724.txt +example_input_file3725.txt +example_input_file3726.txt +example_input_file3727.txt +example_input_file3728.txt +example_input_file3729.txt +example_input_file3730.txt +example_input_file3731.txt +example_input_file3732.txt +example_input_file3733.txt +example_input_file3734.txt +example_input_file3735.txt +example_input_file3736.txt +example_input_file3737.txt +example_input_file3738.txt +example_input_file3739.txt +example_input_file3740.txt +example_input_file3741.txt +example_input_file3742.txt +example_input_file3743.txt +example_input_file3744.txt +example_input_file3745.txt +example_input_file3746.txt +example_input_file3747.txt +example_input_file3748.txt +example_input_file3749.txt +example_input_file3750.txt +example_input_file3751.txt +example_input_file3752.txt +example_input_file3753.txt +example_input_file3754.txt +example_input_file3755.txt +example_input_file3756.txt +example_input_file3757.txt +example_input_file3758.txt +example_input_file3759.txt +example_input_file3760.txt +example_input_file3761.txt +example_input_file3762.txt +example_input_file3763.txt +example_input_file3764.txt +example_input_file3765.txt +example_input_file3766.txt +example_input_file3767.txt +example_input_file3768.txt +example_input_file3769.txt +example_input_file3770.txt +example_input_file3771.txt +example_input_file3772.txt +example_input_file3773.txt +example_input_file3774.txt +example_input_file3775.txt +example_input_file3776.txt +example_input_file3777.txt +example_input_file3778.txt +example_input_file3779.txt +example_input_file3780.txt +example_input_file3781.txt +example_input_file3782.txt +example_input_file3783.txt +example_input_file3784.txt +example_input_file3785.txt +example_input_file3786.txt +example_input_file3787.txt +example_input_file3788.txt +example_input_file3789.txt +example_input_file3790.txt +example_input_file3791.txt +example_input_file3792.txt +example_input_file3793.txt +example_input_file3794.txt +example_input_file3795.txt +example_input_file3796.txt +example_input_file3797.txt +example_input_file3798.txt +example_input_file3799.txt +example_input_file3800.txt +example_input_file3801.txt +example_input_file3802.txt +example_input_file3803.txt +example_input_file3804.txt +example_input_file3805.txt +example_input_file3806.txt +example_input_file3807.txt +example_input_file3808.txt +example_input_file3809.txt +example_input_file3810.txt +example_input_file3811.txt +example_input_file3812.txt +example_input_file3813.txt +example_input_file3814.txt +example_input_file3815.txt +example_input_file3816.txt +example_input_file3817.txt +example_input_file3818.txt +example_input_file3819.txt +example_input_file3820.txt +example_input_file3821.txt +example_input_file3822.txt +example_input_file3823.txt +example_input_file3824.txt +example_input_file3825.txt +example_input_file3826.txt +example_input_file3827.txt +example_input_file3828.txt +example_input_file3829.txt +example_input_file3830.txt +example_input_file3831.txt +example_input_file3832.txt +example_input_file3833.txt +example_input_file3834.txt +example_input_file3835.txt +example_input_file3836.txt +example_input_file3837.txt +example_input_file3838.txt +example_input_file3839.txt +example_input_file3840.txt +example_input_file3841.txt +example_input_file3842.txt +example_input_file3843.txt +example_input_file3844.txt +example_input_file3845.txt +example_input_file3846.txt +example_input_file3847.txt +example_input_file3848.txt +example_input_file3849.txt +example_input_file3850.txt +example_input_file3851.txt +example_input_file3852.txt +example_input_file3853.txt +example_input_file3854.txt +example_input_file3855.txt +example_input_file3856.txt +example_input_file3857.txt +example_input_file3858.txt +example_input_file3859.txt +example_input_file3860.txt +example_input_file3861.txt +example_input_file3862.txt +example_input_file3863.txt +example_input_file3864.txt +example_input_file3865.txt +example_input_file3866.txt +example_input_file3867.txt +example_input_file3868.txt +example_input_file3869.txt +example_input_file3870.txt +example_input_file3871.txt +example_input_file3872.txt +example_input_file3873.txt +example_input_file3874.txt +example_input_file3875.txt +example_input_file3876.txt +example_input_file3877.txt +example_input_file3878.txt +example_input_file3879.txt +example_input_file3880.txt +example_input_file3881.txt +example_input_file3882.txt +example_input_file3883.txt +example_input_file3884.txt +example_input_file3885.txt +example_input_file3886.txt +example_input_file3887.txt +example_input_file3888.txt +example_input_file3889.txt +example_input_file3890.txt +example_input_file3891.txt +example_input_file3892.txt +example_input_file3893.txt +example_input_file3894.txt +example_input_file3895.txt +example_input_file3896.txt +example_input_file3897.txt +example_input_file3898.txt +example_input_file3899.txt +example_input_file3900.txt +example_input_file3901.txt +example_input_file3902.txt +example_input_file3903.txt +example_input_file3904.txt +example_input_file3905.txt +example_input_file3906.txt +example_input_file3907.txt +example_input_file3908.txt +example_input_file3909.txt +example_input_file3910.txt +example_input_file3911.txt +example_input_file3912.txt +example_input_file3913.txt +example_input_file3914.txt +example_input_file3915.txt +example_input_file3916.txt +example_input_file3917.txt +example_input_file3918.txt +example_input_file3919.txt +example_input_file3920.txt +example_input_file3921.txt +example_input_file3922.txt +example_input_file3923.txt +example_input_file3924.txt +example_input_file3925.txt +example_input_file3926.txt +example_input_file3927.txt +example_input_file3928.txt +example_input_file3929.txt +example_input_file3930.txt +example_input_file3931.txt +example_input_file3932.txt +example_input_file3933.txt +example_input_file3934.txt +example_input_file3935.txt +example_input_file3936.txt +example_input_file3937.txt +example_input_file3938.txt +example_input_file3939.txt +example_input_file3940.txt +example_input_file3941.txt +example_input_file3942.txt +example_input_file3943.txt +example_input_file3944.txt +example_input_file3945.txt +example_input_file3946.txt +example_input_file3947.txt +example_input_file3948.txt +example_input_file3949.txt +example_input_file3950.txt +example_input_file3951.txt +example_input_file3952.txt +example_input_file3953.txt +example_input_file3954.txt +example_input_file3955.txt +example_input_file3956.txt +example_input_file3957.txt +example_input_file3958.txt +example_input_file3959.txt +example_input_file3960.txt +example_input_file3961.txt +example_input_file3962.txt +example_input_file3963.txt +example_input_file3964.txt +example_input_file3965.txt +example_input_file3966.txt +example_input_file3967.txt +example_input_file3968.txt +example_input_file3969.txt +example_input_file3970.txt +example_input_file3971.txt +example_input_file3972.txt +example_input_file3973.txt +example_input_file3974.txt +example_input_file3975.txt +example_input_file3976.txt +example_input_file3977.txt +example_input_file3978.txt +example_input_file3979.txt +example_input_file3980.txt +example_input_file3981.txt +example_input_file3982.txt +example_input_file3983.txt +example_input_file3984.txt +example_input_file3985.txt +example_input_file3986.txt +example_input_file3987.txt +example_input_file3988.txt +example_input_file3989.txt +example_input_file3990.txt +example_input_file3991.txt +example_input_file3992.txt +example_input_file3993.txt +example_input_file3994.txt +example_input_file3995.txt +example_input_file3996.txt +example_input_file3997.txt +example_input_file3998.txt +example_input_file3999.txt +example_input_file4000.txt +example_input_file4001.txt +example_input_file4002.txt +example_input_file4003.txt +example_input_file4004.txt +example_input_file4005.txt +example_input_file4006.txt +example_input_file4007.txt +example_input_file4008.txt +example_input_file4009.txt +example_input_file4010.txt +example_input_file4011.txt +example_input_file4012.txt +example_input_file4013.txt +example_input_file4014.txt +example_input_file4015.txt +example_input_file4016.txt +example_input_file4017.txt +example_input_file4018.txt +example_input_file4019.txt +example_input_file4020.txt +example_input_file4021.txt +example_input_file4022.txt +example_input_file4023.txt +example_input_file4024.txt +example_input_file4025.txt +example_input_file4026.txt +example_input_file4027.txt +example_input_file4028.txt +example_input_file4029.txt +example_input_file4030.txt +example_input_file4031.txt +example_input_file4032.txt +example_input_file4033.txt +example_input_file4034.txt +example_input_file4035.txt +example_input_file4036.txt +example_input_file4037.txt +example_input_file4038.txt +example_input_file4039.txt +example_input_file4040.txt +example_input_file4041.txt +example_input_file4042.txt +example_input_file4043.txt +example_input_file4044.txt +example_input_file4045.txt +example_input_file4046.txt +example_input_file4047.txt +example_input_file4048.txt +example_input_file4049.txt +example_input_file4050.txt +example_input_file4051.txt +example_input_file4052.txt +example_input_file4053.txt +example_input_file4054.txt +example_input_file4055.txt +example_input_file4056.txt +example_input_file4057.txt +example_input_file4058.txt +example_input_file4059.txt +example_input_file4060.txt +example_input_file4061.txt +example_input_file4062.txt +example_input_file4063.txt +example_input_file4064.txt +example_input_file4065.txt +example_input_file4066.txt +example_input_file4067.txt +example_input_file4068.txt +example_input_file4069.txt +example_input_file4070.txt +example_input_file4071.txt +example_input_file4072.txt +example_input_file4073.txt +example_input_file4074.txt +example_input_file4075.txt +example_input_file4076.txt +example_input_file4077.txt +example_input_file4078.txt +example_input_file4079.txt +example_input_file4080.txt +example_input_file4081.txt +example_input_file4082.txt +example_input_file4083.txt +example_input_file4084.txt +example_input_file4085.txt +example_input_file4086.txt +example_input_file4087.txt +example_input_file4088.txt +example_input_file4089.txt +example_input_file4090.txt +example_input_file4091.txt +example_input_file4092.txt +example_input_file4093.txt +example_input_file4094.txt +example_input_file4095.txt +example_input_file4096.txt +example_input_file4097.txt +example_input_file4098.txt +example_input_file4099.txt +example_input_file4100.txt +example_input_file4101.txt +example_input_file4102.txt +example_input_file4103.txt +example_input_file4104.txt +example_input_file4105.txt +example_input_file4106.txt +example_input_file4107.txt +example_input_file4108.txt +example_input_file4109.txt +example_input_file4110.txt +example_input_file4111.txt +example_input_file4112.txt +example_input_file4113.txt +example_input_file4114.txt +example_input_file4115.txt +example_input_file4116.txt +example_input_file4117.txt +example_input_file4118.txt +example_input_file4119.txt +example_input_file4120.txt +example_input_file4121.txt +example_input_file4122.txt +example_input_file4123.txt +example_input_file4124.txt +example_input_file4125.txt +example_input_file4126.txt +example_input_file4127.txt +example_input_file4128.txt +example_input_file4129.txt +example_input_file4130.txt +example_input_file4131.txt +example_input_file4132.txt +example_input_file4133.txt +example_input_file4134.txt +example_input_file4135.txt +example_input_file4136.txt +example_input_file4137.txt +example_input_file4138.txt +example_input_file4139.txt +example_input_file4140.txt +example_input_file4141.txt +example_input_file4142.txt +example_input_file4143.txt +example_input_file4144.txt +example_input_file4145.txt +example_input_file4146.txt +example_input_file4147.txt +example_input_file4148.txt +example_input_file4149.txt +example_input_file4150.txt +example_input_file4151.txt +example_input_file4152.txt +example_input_file4153.txt +example_input_file4154.txt +example_input_file4155.txt +example_input_file4156.txt +example_input_file4157.txt +example_input_file4158.txt +example_input_file4159.txt +example_input_file4160.txt +example_input_file4161.txt +example_input_file4162.txt +example_input_file4163.txt +example_input_file4164.txt +example_input_file4165.txt +example_input_file4166.txt +example_input_file4167.txt +example_input_file4168.txt +example_input_file4169.txt +example_input_file4170.txt +example_input_file4171.txt +example_input_file4172.txt +example_input_file4173.txt +example_input_file4174.txt +example_input_file4175.txt +example_input_file4176.txt +example_input_file4177.txt +example_input_file4178.txt +example_input_file4179.txt +example_input_file4180.txt +example_input_file4181.txt +example_input_file4182.txt +example_input_file4183.txt +example_input_file4184.txt +example_input_file4185.txt +example_input_file4186.txt +example_input_file4187.txt +example_input_file4188.txt +example_input_file4189.txt +example_input_file4190.txt +example_input_file4191.txt +example_input_file4192.txt +example_input_file4193.txt +example_input_file4194.txt +example_input_file4195.txt +example_input_file4196.txt +example_input_file4197.txt +example_input_file4198.txt +example_input_file4199.txt +example_input_file4200.txt +example_input_file4201.txt +example_input_file4202.txt +example_input_file4203.txt +example_input_file4204.txt +example_input_file4205.txt +example_input_file4206.txt +example_input_file4207.txt +example_input_file4208.txt +example_input_file4209.txt +example_input_file4210.txt +example_input_file4211.txt +example_input_file4212.txt +example_input_file4213.txt +example_input_file4214.txt +example_input_file4215.txt +example_input_file4216.txt +example_input_file4217.txt +example_input_file4218.txt +example_input_file4219.txt +example_input_file4220.txt +example_input_file4221.txt +example_input_file4222.txt +example_input_file4223.txt +example_input_file4224.txt +example_input_file4225.txt +example_input_file4226.txt +example_input_file4227.txt +example_input_file4228.txt +example_input_file4229.txt +example_input_file4230.txt +example_input_file4231.txt +example_input_file4232.txt +example_input_file4233.txt +example_input_file4234.txt +example_input_file4235.txt +example_input_file4236.txt +example_input_file4237.txt +example_input_file4238.txt +example_input_file4239.txt +example_input_file4240.txt +example_input_file4241.txt +example_input_file4242.txt +example_input_file4243.txt +example_input_file4244.txt +example_input_file4245.txt +example_input_file4246.txt +example_input_file4247.txt +example_input_file4248.txt +example_input_file4249.txt +example_input_file4250.txt +example_input_file4251.txt +example_input_file4252.txt +example_input_file4253.txt +example_input_file4254.txt +example_input_file4255.txt +example_input_file4256.txt +example_input_file4257.txt +example_input_file4258.txt +example_input_file4259.txt +example_input_file4260.txt +example_input_file4261.txt +example_input_file4262.txt +example_input_file4263.txt +example_input_file4264.txt +example_input_file4265.txt +example_input_file4266.txt +example_input_file4267.txt +example_input_file4268.txt +example_input_file4269.txt +example_input_file4270.txt +example_input_file4271.txt +example_input_file4272.txt +example_input_file4273.txt +example_input_file4274.txt +example_input_file4275.txt +example_input_file4276.txt +example_input_file4277.txt +example_input_file4278.txt +example_input_file4279.txt +example_input_file4280.txt +example_input_file4281.txt +example_input_file4282.txt +example_input_file4283.txt +example_input_file4284.txt +example_input_file4285.txt +example_input_file4286.txt +example_input_file4287.txt +example_input_file4288.txt +example_input_file4289.txt +example_input_file4290.txt +example_input_file4291.txt +example_input_file4292.txt +example_input_file4293.txt +example_input_file4294.txt +example_input_file4295.txt +example_input_file4296.txt +example_input_file4297.txt +example_input_file4298.txt +example_input_file4299.txt +example_input_file4300.txt +example_input_file4301.txt +example_input_file4302.txt +example_input_file4303.txt +example_input_file4304.txt +example_input_file4305.txt +example_input_file4306.txt +example_input_file4307.txt +example_input_file4308.txt +example_input_file4309.txt +example_input_file4310.txt +example_input_file4311.txt +example_input_file4312.txt +example_input_file4313.txt +example_input_file4314.txt +example_input_file4315.txt +example_input_file4316.txt +example_input_file4317.txt +example_input_file4318.txt +example_input_file4319.txt +example_input_file4320.txt +example_input_file4321.txt +example_input_file4322.txt +example_input_file4323.txt +example_input_file4324.txt +example_input_file4325.txt +example_input_file4326.txt +example_input_file4327.txt +example_input_file4328.txt +example_input_file4329.txt +example_input_file4330.txt +example_input_file4331.txt +example_input_file4332.txt +example_input_file4333.txt +example_input_file4334.txt +example_input_file4335.txt +example_input_file4336.txt +example_input_file4337.txt +example_input_file4338.txt +example_input_file4339.txt +example_input_file4340.txt +example_input_file4341.txt +example_input_file4342.txt +example_input_file4343.txt +example_input_file4344.txt +example_input_file4345.txt +example_input_file4346.txt +example_input_file4347.txt +example_input_file4348.txt +example_input_file4349.txt +example_input_file4350.txt +example_input_file4351.txt +example_input_file4352.txt +example_input_file4353.txt +example_input_file4354.txt +example_input_file4355.txt +example_input_file4356.txt +example_input_file4357.txt +example_input_file4358.txt +example_input_file4359.txt +example_input_file4360.txt +example_input_file4361.txt +example_input_file4362.txt +example_input_file4363.txt +example_input_file4364.txt +example_input_file4365.txt +example_input_file4366.txt +example_input_file4367.txt +example_input_file4368.txt +example_input_file4369.txt +example_input_file4370.txt +example_input_file4371.txt +example_input_file4372.txt +example_input_file4373.txt +example_input_file4374.txt +example_input_file4375.txt +example_input_file4376.txt +example_input_file4377.txt +example_input_file4378.txt +example_input_file4379.txt +example_input_file4380.txt +example_input_file4381.txt +example_input_file4382.txt +example_input_file4383.txt +example_input_file4384.txt +example_input_file4385.txt +example_input_file4386.txt +example_input_file4387.txt +example_input_file4388.txt +example_input_file4389.txt +example_input_file4390.txt +example_input_file4391.txt +example_input_file4392.txt +example_input_file4393.txt +example_input_file4394.txt +example_input_file4395.txt +example_input_file4396.txt +example_input_file4397.txt +example_input_file4398.txt +example_input_file4399.txt +example_input_file4400.txt +example_input_file4401.txt +example_input_file4402.txt +example_input_file4403.txt +example_input_file4404.txt +example_input_file4405.txt +example_input_file4406.txt +example_input_file4407.txt +example_input_file4408.txt +example_input_file4409.txt +example_input_file4410.txt +example_input_file4411.txt +example_input_file4412.txt +example_input_file4413.txt +example_input_file4414.txt +example_input_file4415.txt +example_input_file4416.txt +example_input_file4417.txt +example_input_file4418.txt +example_input_file4419.txt +example_input_file4420.txt +example_input_file4421.txt +example_input_file4422.txt +example_input_file4423.txt +example_input_file4424.txt +example_input_file4425.txt +example_input_file4426.txt +example_input_file4427.txt +example_input_file4428.txt +example_input_file4429.txt +example_input_file4430.txt +example_input_file4431.txt +example_input_file4432.txt +example_input_file4433.txt +example_input_file4434.txt +example_input_file4435.txt +example_input_file4436.txt +example_input_file4437.txt +example_input_file4438.txt +example_input_file4439.txt +example_input_file4440.txt +example_input_file4441.txt +example_input_file4442.txt +example_input_file4443.txt +example_input_file4444.txt +example_input_file4445.txt +example_input_file4446.txt +example_input_file4447.txt +example_input_file4448.txt +example_input_file4449.txt +example_input_file4450.txt +example_input_file4451.txt +example_input_file4452.txt +example_input_file4453.txt +example_input_file4454.txt +example_input_file4455.txt +example_input_file4456.txt +example_input_file4457.txt +example_input_file4458.txt +example_input_file4459.txt +example_input_file4460.txt +example_input_file4461.txt +example_input_file4462.txt +example_input_file4463.txt +example_input_file4464.txt +example_input_file4465.txt +example_input_file4466.txt +example_input_file4467.txt +example_input_file4468.txt +example_input_file4469.txt +example_input_file4470.txt +example_input_file4471.txt +example_input_file4472.txt +example_input_file4473.txt +example_input_file4474.txt +example_input_file4475.txt +example_input_file4476.txt +example_input_file4477.txt +example_input_file4478.txt +example_input_file4479.txt +example_input_file4480.txt +example_input_file4481.txt +example_input_file4482.txt +example_input_file4483.txt +example_input_file4484.txt +example_input_file4485.txt +example_input_file4486.txt +example_input_file4487.txt +example_input_file4488.txt +example_input_file4489.txt +example_input_file4490.txt +example_input_file4491.txt +example_input_file4492.txt +example_input_file4493.txt +example_input_file4494.txt +example_input_file4495.txt +example_input_file4496.txt +example_input_file4497.txt +example_input_file4498.txt +example_input_file4499.txt +example_input_file4500.txt +example_input_file4501.txt +example_input_file4502.txt +example_input_file4503.txt +example_input_file4504.txt +example_input_file4505.txt +example_input_file4506.txt +example_input_file4507.txt +example_input_file4508.txt +example_input_file4509.txt +example_input_file4510.txt +example_input_file4511.txt +example_input_file4512.txt +example_input_file4513.txt +example_input_file4514.txt +example_input_file4515.txt +example_input_file4516.txt +example_input_file4517.txt +example_input_file4518.txt +example_input_file4519.txt +example_input_file4520.txt +example_input_file4521.txt +example_input_file4522.txt +example_input_file4523.txt +example_input_file4524.txt +example_input_file4525.txt +example_input_file4526.txt +example_input_file4527.txt +example_input_file4528.txt +example_input_file4529.txt +example_input_file4530.txt +example_input_file4531.txt +example_input_file4532.txt +example_input_file4533.txt +example_input_file4534.txt +example_input_file4535.txt +example_input_file4536.txt +example_input_file4537.txt +example_input_file4538.txt +example_input_file4539.txt +example_input_file4540.txt +example_input_file4541.txt +example_input_file4542.txt +example_input_file4543.txt +example_input_file4544.txt +example_input_file4545.txt +example_input_file4546.txt +example_input_file4547.txt +example_input_file4548.txt +example_input_file4549.txt +example_input_file4550.txt +example_input_file4551.txt +example_input_file4552.txt +example_input_file4553.txt +example_input_file4554.txt +example_input_file4555.txt +example_input_file4556.txt +example_input_file4557.txt +example_input_file4558.txt +example_input_file4559.txt +example_input_file4560.txt +example_input_file4561.txt +example_input_file4562.txt +example_input_file4563.txt +example_input_file4564.txt +example_input_file4565.txt +example_input_file4566.txt +example_input_file4567.txt +example_input_file4568.txt +example_input_file4569.txt +example_input_file4570.txt +example_input_file4571.txt +example_input_file4572.txt +example_input_file4573.txt +example_input_file4574.txt +example_input_file4575.txt +example_input_file4576.txt +example_input_file4577.txt +example_input_file4578.txt +example_input_file4579.txt +example_input_file4580.txt +example_input_file4581.txt +example_input_file4582.txt +example_input_file4583.txt +example_input_file4584.txt +example_input_file4585.txt +example_input_file4586.txt +example_input_file4587.txt +example_input_file4588.txt +example_input_file4589.txt +example_input_file4590.txt +example_input_file4591.txt +example_input_file4592.txt +example_input_file4593.txt +example_input_file4594.txt +example_input_file4595.txt +example_input_file4596.txt +example_input_file4597.txt +example_input_file4598.txt +example_input_file4599.txt +example_input_file4600.txt +example_input_file4601.txt +example_input_file4602.txt +example_input_file4603.txt +example_input_file4604.txt +example_input_file4605.txt +example_input_file4606.txt +example_input_file4607.txt +example_input_file4608.txt +example_input_file4609.txt +example_input_file4610.txt +example_input_file4611.txt +example_input_file4612.txt +example_input_file4613.txt +example_input_file4614.txt +example_input_file4615.txt +example_input_file4616.txt +example_input_file4617.txt +example_input_file4618.txt +example_input_file4619.txt +example_input_file4620.txt +example_input_file4621.txt +example_input_file4622.txt +example_input_file4623.txt +example_input_file4624.txt +example_input_file4625.txt +example_input_file4626.txt +example_input_file4627.txt +example_input_file4628.txt +example_input_file4629.txt +example_input_file4630.txt +example_input_file4631.txt +example_input_file4632.txt +example_input_file4633.txt +example_input_file4634.txt +example_input_file4635.txt +example_input_file4636.txt +example_input_file4637.txt +example_input_file4638.txt +example_input_file4639.txt +example_input_file4640.txt +example_input_file4641.txt +example_input_file4642.txt +example_input_file4643.txt +example_input_file4644.txt +example_input_file4645.txt +example_input_file4646.txt +example_input_file4647.txt +example_input_file4648.txt +example_input_file4649.txt +example_input_file4650.txt +example_input_file4651.txt +example_input_file4652.txt +example_input_file4653.txt +example_input_file4654.txt +example_input_file4655.txt +example_input_file4656.txt +example_input_file4657.txt +example_input_file4658.txt +example_input_file4659.txt +example_input_file4660.txt +example_input_file4661.txt +example_input_file4662.txt +example_input_file4663.txt +example_input_file4664.txt +example_input_file4665.txt +example_input_file4666.txt +example_input_file4667.txt +example_input_file4668.txt +example_input_file4669.txt +example_input_file4670.txt +example_input_file4671.txt +example_input_file4672.txt +example_input_file4673.txt +example_input_file4674.txt +example_input_file4675.txt +example_input_file4676.txt +example_input_file4677.txt +example_input_file4678.txt +example_input_file4679.txt +example_input_file4680.txt +example_input_file4681.txt +example_input_file4682.txt +example_input_file4683.txt +example_input_file4684.txt +example_input_file4685.txt +example_input_file4686.txt +example_input_file4687.txt +example_input_file4688.txt +example_input_file4689.txt +example_input_file4690.txt +example_input_file4691.txt +example_input_file4692.txt +example_input_file4693.txt +example_input_file4694.txt +example_input_file4695.txt +example_input_file4696.txt +example_input_file4697.txt +example_input_file4698.txt +example_input_file4699.txt +example_input_file4700.txt +example_input_file4701.txt +example_input_file4702.txt +example_input_file4703.txt +example_input_file4704.txt +example_input_file4705.txt +example_input_file4706.txt +example_input_file4707.txt +example_input_file4708.txt +example_input_file4709.txt +example_input_file4710.txt +example_input_file4711.txt +example_input_file4712.txt +example_input_file4713.txt +example_input_file4714.txt +example_input_file4715.txt +example_input_file4716.txt +example_input_file4717.txt +example_input_file4718.txt +example_input_file4719.txt +example_input_file4720.txt +example_input_file4721.txt +example_input_file4722.txt +example_input_file4723.txt +example_input_file4724.txt +example_input_file4725.txt +example_input_file4726.txt +example_input_file4727.txt +example_input_file4728.txt +example_input_file4729.txt +example_input_file4730.txt +example_input_file4731.txt +example_input_file4732.txt +example_input_file4733.txt +example_input_file4734.txt +example_input_file4735.txt +example_input_file4736.txt +example_input_file4737.txt +example_input_file4738.txt +example_input_file4739.txt +example_input_file4740.txt +example_input_file4741.txt +example_input_file4742.txt +example_input_file4743.txt +example_input_file4744.txt +example_input_file4745.txt +example_input_file4746.txt +example_input_file4747.txt +example_input_file4748.txt +example_input_file4749.txt +example_input_file4750.txt +example_input_file4751.txt +example_input_file4752.txt +example_input_file4753.txt +example_input_file4754.txt +example_input_file4755.txt +example_input_file4756.txt +example_input_file4757.txt +example_input_file4758.txt +example_input_file4759.txt +example_input_file4760.txt +example_input_file4761.txt +example_input_file4762.txt +example_input_file4763.txt +example_input_file4764.txt +example_input_file4765.txt +example_input_file4766.txt +example_input_file4767.txt +example_input_file4768.txt +example_input_file4769.txt +example_input_file4770.txt +example_input_file4771.txt +example_input_file4772.txt +example_input_file4773.txt +example_input_file4774.txt +example_input_file4775.txt +example_input_file4776.txt +example_input_file4777.txt +example_input_file4778.txt +example_input_file4779.txt +example_input_file4780.txt +example_input_file4781.txt +example_input_file4782.txt +example_input_file4783.txt +example_input_file4784.txt +example_input_file4785.txt +example_input_file4786.txt +example_input_file4787.txt +example_input_file4788.txt +example_input_file4789.txt +example_input_file4790.txt +example_input_file4791.txt +example_input_file4792.txt +example_input_file4793.txt +example_input_file4794.txt +example_input_file4795.txt +example_input_file4796.txt +example_input_file4797.txt +example_input_file4798.txt +example_input_file4799.txt +example_input_file4800.txt +example_input_file4801.txt +example_input_file4802.txt +example_input_file4803.txt +example_input_file4804.txt +example_input_file4805.txt +example_input_file4806.txt +example_input_file4807.txt +example_input_file4808.txt +example_input_file4809.txt +example_input_file4810.txt +example_input_file4811.txt +example_input_file4812.txt +example_input_file4813.txt +example_input_file4814.txt +example_input_file4815.txt +example_input_file4816.txt +example_input_file4817.txt +example_input_file4818.txt +example_input_file4819.txt +example_input_file4820.txt +example_input_file4821.txt +example_input_file4822.txt +example_input_file4823.txt +example_input_file4824.txt +example_input_file4825.txt +example_input_file4826.txt +example_input_file4827.txt +example_input_file4828.txt +example_input_file4829.txt +example_input_file4830.txt +example_input_file4831.txt +example_input_file4832.txt +example_input_file4833.txt +example_input_file4834.txt +example_input_file4835.txt +example_input_file4836.txt +example_input_file4837.txt +example_input_file4838.txt +example_input_file4839.txt +example_input_file4840.txt +example_input_file4841.txt +example_input_file4842.txt +example_input_file4843.txt +example_input_file4844.txt +example_input_file4845.txt +example_input_file4846.txt +example_input_file4847.txt +example_input_file4848.txt +example_input_file4849.txt +example_input_file4850.txt +example_input_file4851.txt +example_input_file4852.txt +example_input_file4853.txt +example_input_file4854.txt +example_input_file4855.txt +example_input_file4856.txt +example_input_file4857.txt +example_input_file4858.txt +example_input_file4859.txt +example_input_file4860.txt +example_input_file4861.txt +example_input_file4862.txt +example_input_file4863.txt +example_input_file4864.txt +example_input_file4865.txt +example_input_file4866.txt +example_input_file4867.txt +example_input_file4868.txt +example_input_file4869.txt +example_input_file4870.txt +example_input_file4871.txt +example_input_file4872.txt +example_input_file4873.txt +example_input_file4874.txt +example_input_file4875.txt +example_input_file4876.txt +example_input_file4877.txt +example_input_file4878.txt +example_input_file4879.txt +example_input_file4880.txt +example_input_file4881.txt +example_input_file4882.txt +example_input_file4883.txt +example_input_file4884.txt +example_input_file4885.txt +example_input_file4886.txt +example_input_file4887.txt +example_input_file4888.txt +example_input_file4889.txt +example_input_file4890.txt +example_input_file4891.txt +example_input_file4892.txt +example_input_file4893.txt +example_input_file4894.txt +example_input_file4895.txt +example_input_file4896.txt +example_input_file4897.txt +example_input_file4898.txt +example_input_file4899.txt +example_input_file4900.txt +example_input_file4901.txt +example_input_file4902.txt +example_input_file4903.txt +example_input_file4904.txt +example_input_file4905.txt +example_input_file4906.txt +example_input_file4907.txt +example_input_file4908.txt +example_input_file4909.txt +example_input_file4910.txt +example_input_file4911.txt +example_input_file4912.txt +example_input_file4913.txt +example_input_file4914.txt +example_input_file4915.txt +example_input_file4916.txt +example_input_file4917.txt +example_input_file4918.txt +example_input_file4919.txt +example_input_file4920.txt +example_input_file4921.txt +example_input_file4922.txt +example_input_file4923.txt +example_input_file4924.txt +example_input_file4925.txt +example_input_file4926.txt +example_input_file4927.txt +example_input_file4928.txt +example_input_file4929.txt +example_input_file4930.txt +example_input_file4931.txt +example_input_file4932.txt +example_input_file4933.txt +example_input_file4934.txt +example_input_file4935.txt +example_input_file4936.txt +example_input_file4937.txt +example_input_file4938.txt +example_input_file4939.txt +example_input_file4940.txt +example_input_file4941.txt +example_input_file4942.txt +example_input_file4943.txt +example_input_file4944.txt +example_input_file4945.txt +example_input_file4946.txt +example_input_file4947.txt +example_input_file4948.txt +example_input_file4949.txt +example_input_file4950.txt +example_input_file4951.txt +example_input_file4952.txt +example_input_file4953.txt +example_input_file4954.txt +example_input_file4955.txt +example_input_file4956.txt +example_input_file4957.txt +example_input_file4958.txt +example_input_file4959.txt +example_input_file4960.txt +example_input_file4961.txt +example_input_file4962.txt +example_input_file4963.txt +example_input_file4964.txt +example_input_file4965.txt +example_input_file4966.txt +example_input_file4967.txt +example_input_file4968.txt +example_input_file4969.txt +example_input_file4970.txt +example_input_file4971.txt +example_input_file4972.txt +example_input_file4973.txt +example_input_file4974.txt +example_input_file4975.txt +example_input_file4976.txt +example_input_file4977.txt +example_input_file4978.txt +example_input_file4979.txt +example_input_file4980.txt +example_input_file4981.txt +example_input_file4982.txt +example_input_file4983.txt +example_input_file4984.txt +example_input_file4985.txt +example_input_file4986.txt +example_input_file4987.txt +example_input_file4988.txt +example_input_file4989.txt +example_input_file4990.txt +example_input_file4991.txt +example_input_file4992.txt +example_input_file4993.txt +example_input_file4994.txt +example_input_file4995.txt +example_input_file4996.txt +example_input_file4997.txt +example_input_file4998.txt +example_input_file4999.txt +example_input_file5000.txt +example_input_file5001.txt +example_input_file5002.txt +example_input_file5003.txt +example_input_file5004.txt +example_input_file5005.txt +example_input_file5006.txt +example_input_file5007.txt +example_input_file5008.txt +example_input_file5009.txt +example_input_file5010.txt +example_input_file5011.txt +example_input_file5012.txt +example_input_file5013.txt +example_input_file5014.txt +example_input_file5015.txt +example_input_file5016.txt +example_input_file5017.txt +example_input_file5018.txt +example_input_file5019.txt +example_input_file5020.txt +example_input_file5021.txt +example_input_file5022.txt +example_input_file5023.txt +example_input_file5024.txt +example_input_file5025.txt +example_input_file5026.txt +example_input_file5027.txt +example_input_file5028.txt +example_input_file5029.txt +example_input_file5030.txt +example_input_file5031.txt +example_input_file5032.txt +example_input_file5033.txt +example_input_file5034.txt +example_input_file5035.txt +example_input_file5036.txt +example_input_file5037.txt +example_input_file5038.txt +example_input_file5039.txt +example_input_file5040.txt +example_input_file5041.txt +example_input_file5042.txt +example_input_file5043.txt +example_input_file5044.txt +example_input_file5045.txt +example_input_file5046.txt +example_input_file5047.txt +example_input_file5048.txt +example_input_file5049.txt +example_input_file5050.txt +example_input_file5051.txt +example_input_file5052.txt +example_input_file5053.txt +example_input_file5054.txt +example_input_file5055.txt +example_input_file5056.txt +example_input_file5057.txt +example_input_file5058.txt +example_input_file5059.txt +example_input_file5060.txt +example_input_file5061.txt +example_input_file5062.txt +example_input_file5063.txt +example_input_file5064.txt +example_input_file5065.txt +example_input_file5066.txt +example_input_file5067.txt +example_input_file5068.txt +example_input_file5069.txt +example_input_file5070.txt +example_input_file5071.txt +example_input_file5072.txt +example_input_file5073.txt +example_input_file5074.txt +example_input_file5075.txt +example_input_file5076.txt +example_input_file5077.txt +example_input_file5078.txt +example_input_file5079.txt +example_input_file5080.txt +example_input_file5081.txt +example_input_file5082.txt +example_input_file5083.txt +example_input_file5084.txt +example_input_file5085.txt +example_input_file5086.txt +example_input_file5087.txt +example_input_file5088.txt +example_input_file5089.txt +example_input_file5090.txt +example_input_file5091.txt +example_input_file5092.txt +example_input_file5093.txt +example_input_file5094.txt +example_input_file5095.txt +example_input_file5096.txt +example_input_file5097.txt +example_input_file5098.txt +example_input_file5099.txt +example_input_file5100.txt +example_input_file5101.txt +example_input_file5102.txt +example_input_file5103.txt +example_input_file5104.txt +example_input_file5105.txt +example_input_file5106.txt +example_input_file5107.txt +example_input_file5108.txt +example_input_file5109.txt +example_input_file5110.txt +example_input_file5111.txt +example_input_file5112.txt +example_input_file5113.txt +example_input_file5114.txt +example_input_file5115.txt +example_input_file5116.txt +example_input_file5117.txt +example_input_file5118.txt +example_input_file5119.txt +example_input_file5120.txt +example_input_file5121.txt +example_input_file5122.txt +example_input_file5123.txt +example_input_file5124.txt +example_input_file5125.txt +example_input_file5126.txt +example_input_file5127.txt +example_input_file5128.txt +example_input_file5129.txt +example_input_file5130.txt +example_input_file5131.txt +example_input_file5132.txt +example_input_file5133.txt +example_input_file5134.txt +example_input_file5135.txt +example_input_file5136.txt +example_input_file5137.txt +example_input_file5138.txt +example_input_file5139.txt +example_input_file5140.txt +example_input_file5141.txt +example_input_file5142.txt +example_input_file5143.txt +example_input_file5144.txt +example_input_file5145.txt +example_input_file5146.txt +example_input_file5147.txt +example_input_file5148.txt +example_input_file5149.txt +example_input_file5150.txt +example_input_file5151.txt +example_input_file5152.txt +example_input_file5153.txt +example_input_file5154.txt +example_input_file5155.txt +example_input_file5156.txt +example_input_file5157.txt +example_input_file5158.txt +example_input_file5159.txt +example_input_file5160.txt +example_input_file5161.txt +example_input_file5162.txt +example_input_file5163.txt +example_input_file5164.txt +example_input_file5165.txt +example_input_file5166.txt +example_input_file5167.txt +example_input_file5168.txt +example_input_file5169.txt +example_input_file5170.txt +example_input_file5171.txt +example_input_file5172.txt +example_input_file5173.txt +example_input_file5174.txt +example_input_file5175.txt +example_input_file5176.txt +example_input_file5177.txt +example_input_file5178.txt +example_input_file5179.txt +example_input_file5180.txt +example_input_file5181.txt +example_input_file5182.txt +example_input_file5183.txt +example_input_file5184.txt +example_input_file5185.txt +example_input_file5186.txt +example_input_file5187.txt +example_input_file5188.txt +example_input_file5189.txt +example_input_file5190.txt +example_input_file5191.txt +example_input_file5192.txt +example_input_file5193.txt +example_input_file5194.txt +example_input_file5195.txt +example_input_file5196.txt +example_input_file5197.txt +example_input_file5198.txt +example_input_file5199.txt +example_input_file5200.txt +example_input_file5201.txt +example_input_file5202.txt +example_input_file5203.txt +example_input_file5204.txt +example_input_file5205.txt +example_input_file5206.txt +example_input_file5207.txt +example_input_file5208.txt +example_input_file5209.txt +example_input_file5210.txt +example_input_file5211.txt +example_input_file5212.txt +example_input_file5213.txt +example_input_file5214.txt +example_input_file5215.txt +example_input_file5216.txt +example_input_file5217.txt +example_input_file5218.txt +example_input_file5219.txt +example_input_file5220.txt +example_input_file5221.txt +example_input_file5222.txt +example_input_file5223.txt +example_input_file5224.txt +example_input_file5225.txt +example_input_file5226.txt +example_input_file5227.txt +example_input_file5228.txt +example_input_file5229.txt +example_input_file5230.txt +example_input_file5231.txt +example_input_file5232.txt +example_input_file5233.txt +example_input_file5234.txt +example_input_file5235.txt +example_input_file5236.txt +example_input_file5237.txt +example_input_file5238.txt +example_input_file5239.txt +example_input_file5240.txt +example_input_file5241.txt +example_input_file5242.txt +example_input_file5243.txt +example_input_file5244.txt +example_input_file5245.txt +example_input_file5246.txt +example_input_file5247.txt +example_input_file5248.txt +example_input_file5249.txt +example_input_file5250.txt +example_input_file5251.txt +example_input_file5252.txt +example_input_file5253.txt +example_input_file5254.txt +example_input_file5255.txt +example_input_file5256.txt +example_input_file5257.txt +example_input_file5258.txt +example_input_file5259.txt +example_input_file5260.txt +example_input_file5261.txt +example_input_file5262.txt +example_input_file5263.txt +example_input_file5264.txt +example_input_file5265.txt +example_input_file5266.txt +example_input_file5267.txt +example_input_file5268.txt +example_input_file5269.txt +example_input_file5270.txt +example_input_file5271.txt +example_input_file5272.txt +example_input_file5273.txt +example_input_file5274.txt +example_input_file5275.txt +example_input_file5276.txt +example_input_file5277.txt +example_input_file5278.txt +example_input_file5279.txt +example_input_file5280.txt +example_input_file5281.txt +example_input_file5282.txt +example_input_file5283.txt +example_input_file5284.txt +example_input_file5285.txt +example_input_file5286.txt +example_input_file5287.txt +example_input_file5288.txt +example_input_file5289.txt +example_input_file5290.txt +example_input_file5291.txt +example_input_file5292.txt +example_input_file5293.txt +example_input_file5294.txt +example_input_file5295.txt +example_input_file5296.txt +example_input_file5297.txt +example_input_file5298.txt +example_input_file5299.txt +example_input_file5300.txt +example_input_file5301.txt +example_input_file5302.txt +example_input_file5303.txt +example_input_file5304.txt +example_input_file5305.txt +example_input_file5306.txt +example_input_file5307.txt +example_input_file5308.txt +example_input_file5309.txt +example_input_file5310.txt +example_input_file5311.txt +example_input_file5312.txt +example_input_file5313.txt +example_input_file5314.txt +example_input_file5315.txt +example_input_file5316.txt +example_input_file5317.txt +example_input_file5318.txt +example_input_file5319.txt +example_input_file5320.txt +example_input_file5321.txt +example_input_file5322.txt +example_input_file5323.txt +example_input_file5324.txt +example_input_file5325.txt +example_input_file5326.txt +example_input_file5327.txt +example_input_file5328.txt +example_input_file5329.txt +example_input_file5330.txt +example_input_file5331.txt +example_input_file5332.txt +example_input_file5333.txt +example_input_file5334.txt +example_input_file5335.txt +example_input_file5336.txt +example_input_file5337.txt +example_input_file5338.txt +example_input_file5339.txt +example_input_file5340.txt +example_input_file5341.txt +example_input_file5342.txt +example_input_file5343.txt +example_input_file5344.txt +example_input_file5345.txt +example_input_file5346.txt +example_input_file5347.txt +example_input_file5348.txt +example_input_file5349.txt +example_input_file5350.txt +example_input_file5351.txt +example_input_file5352.txt +example_input_file5353.txt +example_input_file5354.txt +example_input_file5355.txt +example_input_file5356.txt +example_input_file5357.txt +example_input_file5358.txt +example_input_file5359.txt +example_input_file5360.txt +example_input_file5361.txt +example_input_file5362.txt +example_input_file5363.txt +example_input_file5364.txt +example_input_file5365.txt +example_input_file5366.txt +example_input_file5367.txt +example_input_file5368.txt +example_input_file5369.txt +example_input_file5370.txt +example_input_file5371.txt +example_input_file5372.txt +example_input_file5373.txt +example_input_file5374.txt +example_input_file5375.txt +example_input_file5376.txt +example_input_file5377.txt +example_input_file5378.txt +example_input_file5379.txt +example_input_file5380.txt +example_input_file5381.txt +example_input_file5382.txt +example_input_file5383.txt +example_input_file5384.txt +example_input_file5385.txt +example_input_file5386.txt +example_input_file5387.txt +example_input_file5388.txt +example_input_file5389.txt +example_input_file5390.txt +example_input_file5391.txt +example_input_file5392.txt +example_input_file5393.txt +example_input_file5394.txt +example_input_file5395.txt +example_input_file5396.txt +example_input_file5397.txt +example_input_file5398.txt +example_input_file5399.txt +example_input_file5400.txt +example_input_file5401.txt +example_input_file5402.txt +example_input_file5403.txt +example_input_file5404.txt +example_input_file5405.txt +example_input_file5406.txt +example_input_file5407.txt +example_input_file5408.txt +example_input_file5409.txt +example_input_file5410.txt +example_input_file5411.txt +example_input_file5412.txt +example_input_file5413.txt +example_input_file5414.txt +example_input_file5415.txt +example_input_file5416.txt +example_input_file5417.txt +example_input_file5418.txt +example_input_file5419.txt +example_input_file5420.txt +example_input_file5421.txt +example_input_file5422.txt +example_input_file5423.txt +example_input_file5424.txt +example_input_file5425.txt +example_input_file5426.txt +example_input_file5427.txt +example_input_file5428.txt +example_input_file5429.txt +example_input_file5430.txt +example_input_file5431.txt +example_input_file5432.txt +example_input_file5433.txt +example_input_file5434.txt +example_input_file5435.txt +example_input_file5436.txt +example_input_file5437.txt +example_input_file5438.txt +example_input_file5439.txt +example_input_file5440.txt +example_input_file5441.txt +example_input_file5442.txt +example_input_file5443.txt +example_input_file5444.txt +example_input_file5445.txt +example_input_file5446.txt +example_input_file5447.txt +example_input_file5448.txt +example_input_file5449.txt +example_input_file5450.txt +example_input_file5451.txt +example_input_file5452.txt +example_input_file5453.txt +example_input_file5454.txt +example_input_file5455.txt +example_input_file5456.txt +example_input_file5457.txt +example_input_file5458.txt +example_input_file5459.txt +example_input_file5460.txt +example_input_file5461.txt +example_input_file5462.txt +example_input_file5463.txt +example_input_file5464.txt +example_input_file5465.txt +example_input_file5466.txt +example_input_file5467.txt +example_input_file5468.txt +example_input_file5469.txt +example_input_file5470.txt +example_input_file5471.txt +example_input_file5472.txt +example_input_file5473.txt +example_input_file5474.txt +example_input_file5475.txt +example_input_file5476.txt +example_input_file5477.txt +example_input_file5478.txt +example_input_file5479.txt +example_input_file5480.txt +example_input_file5481.txt +example_input_file5482.txt +example_input_file5483.txt +example_input_file5484.txt +example_input_file5485.txt +example_input_file5486.txt +example_input_file5487.txt +example_input_file5488.txt +example_input_file5489.txt +example_input_file5490.txt +example_input_file5491.txt +example_input_file5492.txt +example_input_file5493.txt +example_input_file5494.txt +example_input_file5495.txt +example_input_file5496.txt +example_input_file5497.txt +example_input_file5498.txt +example_input_file5499.txt +example_input_file5500.txt +example_input_file5501.txt +example_input_file5502.txt +example_input_file5503.txt +example_input_file5504.txt +example_input_file5505.txt +example_input_file5506.txt +example_input_file5507.txt +example_input_file5508.txt +example_input_file5509.txt +example_input_file5510.txt +example_input_file5511.txt +example_input_file5512.txt +example_input_file5513.txt +example_input_file5514.txt +example_input_file5515.txt +example_input_file5516.txt +example_input_file5517.txt +example_input_file5518.txt +example_input_file5519.txt +example_input_file5520.txt +example_input_file5521.txt +example_input_file5522.txt +example_input_file5523.txt +example_input_file5524.txt +example_input_file5525.txt +example_input_file5526.txt +example_input_file5527.txt +example_input_file5528.txt +example_input_file5529.txt +example_input_file5530.txt +example_input_file5531.txt +example_input_file5532.txt +example_input_file5533.txt +example_input_file5534.txt +example_input_file5535.txt +example_input_file5536.txt +example_input_file5537.txt +example_input_file5538.txt +example_input_file5539.txt +example_input_file5540.txt +example_input_file5541.txt +example_input_file5542.txt +example_input_file5543.txt +example_input_file5544.txt +example_input_file5545.txt +example_input_file5546.txt +example_input_file5547.txt +example_input_file5548.txt +example_input_file5549.txt +example_input_file5550.txt +example_input_file5551.txt +example_input_file5552.txt +example_input_file5553.txt +example_input_file5554.txt +example_input_file5555.txt +example_input_file5556.txt +example_input_file5557.txt +example_input_file5558.txt +example_input_file5559.txt +example_input_file5560.txt +example_input_file5561.txt +example_input_file5562.txt +example_input_file5563.txt +example_input_file5564.txt +example_input_file5565.txt +example_input_file5566.txt +example_input_file5567.txt +example_input_file5568.txt +example_input_file5569.txt +example_input_file5570.txt +example_input_file5571.txt +example_input_file5572.txt +example_input_file5573.txt +example_input_file5574.txt +example_input_file5575.txt +example_input_file5576.txt +example_input_file5577.txt +example_input_file5578.txt +example_input_file5579.txt +example_input_file5580.txt +example_input_file5581.txt +example_input_file5582.txt +example_input_file5583.txt +example_input_file5584.txt +example_input_file5585.txt +example_input_file5586.txt +example_input_file5587.txt +example_input_file5588.txt +example_input_file5589.txt +example_input_file5590.txt +example_input_file5591.txt +example_input_file5592.txt +example_input_file5593.txt +example_input_file5594.txt +example_input_file5595.txt +example_input_file5596.txt +example_input_file5597.txt +example_input_file5598.txt +example_input_file5599.txt +example_input_file5600.txt +example_input_file5601.txt +example_input_file5602.txt +example_input_file5603.txt +example_input_file5604.txt +example_input_file5605.txt +example_input_file5606.txt +example_input_file5607.txt +example_input_file5608.txt +example_input_file5609.txt +example_input_file5610.txt +example_input_file5611.txt +example_input_file5612.txt +example_input_file5613.txt +example_input_file5614.txt +example_input_file5615.txt +example_input_file5616.txt +example_input_file5617.txt +example_input_file5618.txt +example_input_file5619.txt +example_input_file5620.txt +example_input_file5621.txt +example_input_file5622.txt +example_input_file5623.txt +example_input_file5624.txt +example_input_file5625.txt +example_input_file5626.txt +example_input_file5627.txt +example_input_file5628.txt +example_input_file5629.txt +example_input_file5630.txt +example_input_file5631.txt +example_input_file5632.txt +example_input_file5633.txt +example_input_file5634.txt +example_input_file5635.txt +example_input_file5636.txt +example_input_file5637.txt +example_input_file5638.txt +example_input_file5639.txt +example_input_file5640.txt +example_input_file5641.txt +example_input_file5642.txt +example_input_file5643.txt +example_input_file5644.txt +example_input_file5645.txt +example_input_file5646.txt +example_input_file5647.txt +example_input_file5648.txt +example_input_file5649.txt +example_input_file5650.txt +example_input_file5651.txt +example_input_file5652.txt +example_input_file5653.txt +example_input_file5654.txt +example_input_file5655.txt +example_input_file5656.txt +example_input_file5657.txt +example_input_file5658.txt +example_input_file5659.txt +example_input_file5660.txt +example_input_file5661.txt +example_input_file5662.txt +example_input_file5663.txt +example_input_file5664.txt +example_input_file5665.txt +example_input_file5666.txt +example_input_file5667.txt +example_input_file5668.txt +example_input_file5669.txt +example_input_file5670.txt +example_input_file5671.txt +example_input_file5672.txt +example_input_file5673.txt +example_input_file5674.txt +example_input_file5675.txt +example_input_file5676.txt +example_input_file5677.txt +example_input_file5678.txt +example_input_file5679.txt +example_input_file5680.txt +example_input_file5681.txt +example_input_file5682.txt +example_input_file5683.txt +example_input_file5684.txt +example_input_file5685.txt +example_input_file5686.txt +example_input_file5687.txt +example_input_file5688.txt +example_input_file5689.txt +example_input_file5690.txt +example_input_file5691.txt +example_input_file5692.txt +example_input_file5693.txt +example_input_file5694.txt +example_input_file5695.txt +example_input_file5696.txt +example_input_file5697.txt +example_input_file5698.txt +example_input_file5699.txt +example_input_file5700.txt +example_input_file5701.txt +example_input_file5702.txt +example_input_file5703.txt +example_input_file5704.txt +example_input_file5705.txt +example_input_file5706.txt +example_input_file5707.txt +example_input_file5708.txt +example_input_file5709.txt +example_input_file5710.txt +example_input_file5711.txt +example_input_file5712.txt +example_input_file5713.txt +example_input_file5714.txt +example_input_file5715.txt +example_input_file5716.txt +example_input_file5717.txt +example_input_file5718.txt +example_input_file5719.txt +example_input_file5720.txt +example_input_file5721.txt +example_input_file5722.txt +example_input_file5723.txt +example_input_file5724.txt +example_input_file5725.txt +example_input_file5726.txt +example_input_file5727.txt +example_input_file5728.txt +example_input_file5729.txt +example_input_file5730.txt +example_input_file5731.txt +example_input_file5732.txt +example_input_file5733.txt +example_input_file5734.txt +example_input_file5735.txt +example_input_file5736.txt +example_input_file5737.txt +example_input_file5738.txt +example_input_file5739.txt +example_input_file5740.txt +example_input_file5741.txt +example_input_file5742.txt +example_input_file5743.txt +example_input_file5744.txt +example_input_file5745.txt +example_input_file5746.txt +example_input_file5747.txt +example_input_file5748.txt +example_input_file5749.txt +example_input_file5750.txt +example_input_file5751.txt +example_input_file5752.txt +example_input_file5753.txt +example_input_file5754.txt +example_input_file5755.txt +example_input_file5756.txt +example_input_file5757.txt +example_input_file5758.txt +example_input_file5759.txt +example_input_file5760.txt +example_input_file5761.txt +example_input_file5762.txt +example_input_file5763.txt +example_input_file5764.txt +example_input_file5765.txt +example_input_file5766.txt +example_input_file5767.txt +example_input_file5768.txt +example_input_file5769.txt +example_input_file5770.txt +example_input_file5771.txt +example_input_file5772.txt +example_input_file5773.txt +example_input_file5774.txt +example_input_file5775.txt +example_input_file5776.txt +example_input_file5777.txt +example_input_file5778.txt +example_input_file5779.txt +example_input_file5780.txt +example_input_file5781.txt +example_input_file5782.txt +example_input_file5783.txt +example_input_file5784.txt +example_input_file5785.txt +example_input_file5786.txt +example_input_file5787.txt +example_input_file5788.txt +example_input_file5789.txt +example_input_file5790.txt +example_input_file5791.txt +example_input_file5792.txt +example_input_file5793.txt +example_input_file5794.txt +example_input_file5795.txt +example_input_file5796.txt +example_input_file5797.txt +example_input_file5798.txt +example_input_file5799.txt +example_input_file5800.txt +example_input_file5801.txt +example_input_file5802.txt +example_input_file5803.txt +example_input_file5804.txt +example_input_file5805.txt +example_input_file5806.txt +example_input_file5807.txt +example_input_file5808.txt +example_input_file5809.txt +example_input_file5810.txt +example_input_file5811.txt +example_input_file5812.txt +example_input_file5813.txt +example_input_file5814.txt +example_input_file5815.txt +example_input_file5816.txt +example_input_file5817.txt +example_input_file5818.txt +example_input_file5819.txt +example_input_file5820.txt +example_input_file5821.txt +example_input_file5822.txt +example_input_file5823.txt +example_input_file5824.txt +example_input_file5825.txt +example_input_file5826.txt +example_input_file5827.txt +example_input_file5828.txt +example_input_file5829.txt +example_input_file5830.txt +example_input_file5831.txt +example_input_file5832.txt +example_input_file5833.txt +example_input_file5834.txt +example_input_file5835.txt +example_input_file5836.txt +example_input_file5837.txt +example_input_file5838.txt +example_input_file5839.txt +example_input_file5840.txt +example_input_file5841.txt +example_input_file5842.txt +example_input_file5843.txt +example_input_file5844.txt +example_input_file5845.txt +example_input_file5846.txt +example_input_file5847.txt +example_input_file5848.txt +example_input_file5849.txt +example_input_file5850.txt +example_input_file5851.txt +example_input_file5852.txt +example_input_file5853.txt +example_input_file5854.txt +example_input_file5855.txt +example_input_file5856.txt +example_input_file5857.txt +example_input_file5858.txt +example_input_file5859.txt +example_input_file5860.txt +example_input_file5861.txt +example_input_file5862.txt +example_input_file5863.txt +example_input_file5864.txt +example_input_file5865.txt +example_input_file5866.txt +example_input_file5867.txt +example_input_file5868.txt +example_input_file5869.txt +example_input_file5870.txt +example_input_file5871.txt +example_input_file5872.txt +example_input_file5873.txt +example_input_file5874.txt +example_input_file5875.txt +example_input_file5876.txt +example_input_file5877.txt +example_input_file5878.txt +example_input_file5879.txt +example_input_file5880.txt +example_input_file5881.txt +example_input_file5882.txt +example_input_file5883.txt +example_input_file5884.txt +example_input_file5885.txt +example_input_file5886.txt +example_input_file5887.txt +example_input_file5888.txt +example_input_file5889.txt +example_input_file5890.txt +example_input_file5891.txt +example_input_file5892.txt +example_input_file5893.txt +example_input_file5894.txt +example_input_file5895.txt +example_input_file5896.txt +example_input_file5897.txt +example_input_file5898.txt +example_input_file5899.txt +example_input_file5900.txt +example_input_file5901.txt +example_input_file5902.txt +example_input_file5903.txt +example_input_file5904.txt +example_input_file5905.txt +example_input_file5906.txt +example_input_file5907.txt +example_input_file5908.txt +example_input_file5909.txt +example_input_file5910.txt +example_input_file5911.txt +example_input_file5912.txt +example_input_file5913.txt +example_input_file5914.txt +example_input_file5915.txt +example_input_file5916.txt +example_input_file5917.txt +example_input_file5918.txt +example_input_file5919.txt +example_input_file5920.txt +example_input_file5921.txt +example_input_file5922.txt +example_input_file5923.txt +example_input_file5924.txt +example_input_file5925.txt +example_input_file5926.txt +example_input_file5927.txt +example_input_file5928.txt +example_input_file5929.txt +example_input_file5930.txt +example_input_file5931.txt +example_input_file5932.txt +example_input_file5933.txt +example_input_file5934.txt +example_input_file5935.txt +example_input_file5936.txt +example_input_file5937.txt +example_input_file5938.txt +example_input_file5939.txt +example_input_file5940.txt +example_input_file5941.txt +example_input_file5942.txt +example_input_file5943.txt +example_input_file5944.txt +example_input_file5945.txt +example_input_file5946.txt +example_input_file5947.txt +example_input_file5948.txt +example_input_file5949.txt +example_input_file5950.txt +example_input_file5951.txt +example_input_file5952.txt +example_input_file5953.txt +example_input_file5954.txt +example_input_file5955.txt +example_input_file5956.txt +example_input_file5957.txt +example_input_file5958.txt +example_input_file5959.txt +example_input_file5960.txt +example_input_file5961.txt +example_input_file5962.txt +example_input_file5963.txt +example_input_file5964.txt +example_input_file5965.txt +example_input_file5966.txt +example_input_file5967.txt +example_input_file5968.txt +example_input_file5969.txt +example_input_file5970.txt +example_input_file5971.txt +example_input_file5972.txt +example_input_file5973.txt +example_input_file5974.txt +example_input_file5975.txt +example_input_file5976.txt +example_input_file5977.txt +example_input_file5978.txt +example_input_file5979.txt +example_input_file5980.txt +example_input_file5981.txt +example_input_file5982.txt +example_input_file5983.txt +example_input_file5984.txt +example_input_file5985.txt +example_input_file5986.txt +example_input_file5987.txt +example_input_file5988.txt +example_input_file5989.txt +example_input_file5990.txt +example_input_file5991.txt +example_input_file5992.txt +example_input_file5993.txt +example_input_file5994.txt +example_input_file5995.txt +example_input_file5996.txt +example_input_file5997.txt +example_input_file5998.txt +example_input_file5999.txt +example_input_file6000.txt +example_input_file6001.txt +example_input_file6002.txt +example_input_file6003.txt +example_input_file6004.txt +example_input_file6005.txt +example_input_file6006.txt +example_input_file6007.txt +example_input_file6008.txt +example_input_file6009.txt +example_input_file6010.txt +example_input_file6011.txt +example_input_file6012.txt +example_input_file6013.txt +example_input_file6014.txt +example_input_file6015.txt +example_input_file6016.txt +example_input_file6017.txt +example_input_file6018.txt +example_input_file6019.txt +example_input_file6020.txt +example_input_file6021.txt +example_input_file6022.txt +example_input_file6023.txt +example_input_file6024.txt +example_input_file6025.txt +example_input_file6026.txt +example_input_file6027.txt +example_input_file6028.txt +example_input_file6029.txt +example_input_file6030.txt +example_input_file6031.txt +example_input_file6032.txt +example_input_file6033.txt +example_input_file6034.txt +example_input_file6035.txt +example_input_file6036.txt +example_input_file6037.txt +example_input_file6038.txt +example_input_file6039.txt +example_input_file6040.txt +example_input_file6041.txt +example_input_file6042.txt +example_input_file6043.txt +example_input_file6044.txt +example_input_file6045.txt +example_input_file6046.txt +example_input_file6047.txt +example_input_file6048.txt +example_input_file6049.txt +example_input_file6050.txt +example_input_file6051.txt +example_input_file6052.txt +example_input_file6053.txt +example_input_file6054.txt +example_input_file6055.txt +example_input_file6056.txt +example_input_file6057.txt +example_input_file6058.txt +example_input_file6059.txt +example_input_file6060.txt +example_input_file6061.txt +example_input_file6062.txt +example_input_file6063.txt +example_input_file6064.txt +example_input_file6065.txt +example_input_file6066.txt +example_input_file6067.txt +example_input_file6068.txt +example_input_file6069.txt +example_input_file6070.txt +example_input_file6071.txt +example_input_file6072.txt +example_input_file6073.txt +example_input_file6074.txt +example_input_file6075.txt +example_input_file6076.txt +example_input_file6077.txt +example_input_file6078.txt +example_input_file6079.txt +example_input_file6080.txt +example_input_file6081.txt +example_input_file6082.txt +example_input_file6083.txt +example_input_file6084.txt +example_input_file6085.txt +example_input_file6086.txt +example_input_file6087.txt +example_input_file6088.txt +example_input_file6089.txt +example_input_file6090.txt +example_input_file6091.txt +example_input_file6092.txt +example_input_file6093.txt +example_input_file6094.txt +example_input_file6095.txt +example_input_file6096.txt +example_input_file6097.txt +example_input_file6098.txt +example_input_file6099.txt +example_input_file6100.txt +example_input_file6101.txt +example_input_file6102.txt +example_input_file6103.txt +example_input_file6104.txt +example_input_file6105.txt +example_input_file6106.txt +example_input_file6107.txt +example_input_file6108.txt +example_input_file6109.txt +example_input_file6110.txt +example_input_file6111.txt +example_input_file6112.txt +example_input_file6113.txt +example_input_file6114.txt +example_input_file6115.txt +example_input_file6116.txt +example_input_file6117.txt +example_input_file6118.txt +example_input_file6119.txt +example_input_file6120.txt +example_input_file6121.txt +example_input_file6122.txt +example_input_file6123.txt +example_input_file6124.txt +example_input_file6125.txt +example_input_file6126.txt +example_input_file6127.txt +example_input_file6128.txt +example_input_file6129.txt +example_input_file6130.txt +example_input_file6131.txt +example_input_file6132.txt +example_input_file6133.txt +example_input_file6134.txt +example_input_file6135.txt +example_input_file6136.txt +example_input_file6137.txt +example_input_file6138.txt +example_input_file6139.txt +example_input_file6140.txt +example_input_file6141.txt +example_input_file6142.txt +example_input_file6143.txt +example_input_file6144.txt +example_input_file6145.txt +example_input_file6146.txt +example_input_file6147.txt +example_input_file6148.txt +example_input_file6149.txt +example_input_file6150.txt +example_input_file6151.txt +example_input_file6152.txt +example_input_file6153.txt +example_input_file6154.txt +example_input_file6155.txt +example_input_file6156.txt +example_input_file6157.txt +example_input_file6158.txt +example_input_file6159.txt +example_input_file6160.txt +example_input_file6161.txt +example_input_file6162.txt +example_input_file6163.txt +example_input_file6164.txt +example_input_file6165.txt +example_input_file6166.txt +example_input_file6167.txt +example_input_file6168.txt +example_input_file6169.txt +example_input_file6170.txt +example_input_file6171.txt +example_input_file6172.txt +example_input_file6173.txt +example_input_file6174.txt +example_input_file6175.txt +example_input_file6176.txt +example_input_file6177.txt +example_input_file6178.txt +example_input_file6179.txt +example_input_file6180.txt +example_input_file6181.txt +example_input_file6182.txt +example_input_file6183.txt +example_input_file6184.txt +example_input_file6185.txt +example_input_file6186.txt +example_input_file6187.txt +example_input_file6188.txt +example_input_file6189.txt +example_input_file6190.txt +example_input_file6191.txt +example_input_file6192.txt +example_input_file6193.txt +example_input_file6194.txt +example_input_file6195.txt +example_input_file6196.txt +example_input_file6197.txt +example_input_file6198.txt +example_input_file6199.txt +example_input_file6200.txt +example_input_file6201.txt +example_input_file6202.txt +example_input_file6203.txt +example_input_file6204.txt +example_input_file6205.txt +example_input_file6206.txt +example_input_file6207.txt +example_input_file6208.txt +example_input_file6209.txt +example_input_file6210.txt +example_input_file6211.txt +example_input_file6212.txt +example_input_file6213.txt +example_input_file6214.txt +example_input_file6215.txt +example_input_file6216.txt +example_input_file6217.txt +example_input_file6218.txt +example_input_file6219.txt +example_input_file6220.txt +example_input_file6221.txt +example_input_file6222.txt +example_input_file6223.txt +example_input_file6224.txt +example_input_file6225.txt +example_input_file6226.txt +example_input_file6227.txt +example_input_file6228.txt +example_input_file6229.txt +example_input_file6230.txt +example_input_file6231.txt +example_input_file6232.txt +example_input_file6233.txt +example_input_file6234.txt +example_input_file6235.txt +example_input_file6236.txt +example_input_file6237.txt +example_input_file6238.txt +example_input_file6239.txt +example_input_file6240.txt +example_input_file6241.txt +example_input_file6242.txt +example_input_file6243.txt +example_input_file6244.txt +example_input_file6245.txt +example_input_file6246.txt +example_input_file6247.txt +example_input_file6248.txt +example_input_file6249.txt +example_input_file6250.txt +example_input_file6251.txt +example_input_file6252.txt +example_input_file6253.txt +example_input_file6254.txt +example_input_file6255.txt +example_input_file6256.txt +example_input_file6257.txt +example_input_file6258.txt +example_input_file6259.txt +example_input_file6260.txt +example_input_file6261.txt +example_input_file6262.txt +example_input_file6263.txt +example_input_file6264.txt +example_input_file6265.txt +example_input_file6266.txt +example_input_file6267.txt +example_input_file6268.txt +example_input_file6269.txt +example_input_file6270.txt +example_input_file6271.txt +example_input_file6272.txt +example_input_file6273.txt +example_input_file6274.txt +example_input_file6275.txt +example_input_file6276.txt +example_input_file6277.txt +example_input_file6278.txt +example_input_file6279.txt +example_input_file6280.txt +example_input_file6281.txt +example_input_file6282.txt +example_input_file6283.txt +example_input_file6284.txt +example_input_file6285.txt +example_input_file6286.txt +example_input_file6287.txt +example_input_file6288.txt +example_input_file6289.txt +example_input_file6290.txt +example_input_file6291.txt +example_input_file6292.txt +example_input_file6293.txt +example_input_file6294.txt +example_input_file6295.txt +example_input_file6296.txt +example_input_file6297.txt +example_input_file6298.txt +example_input_file6299.txt +example_input_file6300.txt +example_input_file6301.txt +example_input_file6302.txt +example_input_file6303.txt +example_input_file6304.txt +example_input_file6305.txt +example_input_file6306.txt +example_input_file6307.txt +example_input_file6308.txt +example_input_file6309.txt +example_input_file6310.txt +example_input_file6311.txt +example_input_file6312.txt +example_input_file6313.txt +example_input_file6314.txt +example_input_file6315.txt +example_input_file6316.txt +example_input_file6317.txt +example_input_file6318.txt +example_input_file6319.txt +example_input_file6320.txt +example_input_file6321.txt +example_input_file6322.txt +example_input_file6323.txt +example_input_file6324.txt +example_input_file6325.txt +example_input_file6326.txt +example_input_file6327.txt +example_input_file6328.txt +example_input_file6329.txt +example_input_file6330.txt +example_input_file6331.txt +example_input_file6332.txt +example_input_file6333.txt +example_input_file6334.txt +example_input_file6335.txt +example_input_file6336.txt +example_input_file6337.txt +example_input_file6338.txt +example_input_file6339.txt +example_input_file6340.txt +example_input_file6341.txt +example_input_file6342.txt +example_input_file6343.txt +example_input_file6344.txt +example_input_file6345.txt +example_input_file6346.txt +example_input_file6347.txt +example_input_file6348.txt +example_input_file6349.txt +example_input_file6350.txt +example_input_file6351.txt +example_input_file6352.txt +example_input_file6353.txt +example_input_file6354.txt +example_input_file6355.txt +example_input_file6356.txt +example_input_file6357.txt +example_input_file6358.txt +example_input_file6359.txt +example_input_file6360.txt +example_input_file6361.txt +example_input_file6362.txt +example_input_file6363.txt +example_input_file6364.txt +example_input_file6365.txt +example_input_file6366.txt +example_input_file6367.txt +example_input_file6368.txt +example_input_file6369.txt +example_input_file6370.txt +example_input_file6371.txt +example_input_file6372.txt +example_input_file6373.txt +example_input_file6374.txt +example_input_file6375.txt +example_input_file6376.txt +example_input_file6377.txt +example_input_file6378.txt +example_input_file6379.txt +example_input_file6380.txt +example_input_file6381.txt +example_input_file6382.txt +example_input_file6383.txt +example_input_file6384.txt +example_input_file6385.txt +example_input_file6386.txt +example_input_file6387.txt +example_input_file6388.txt +example_input_file6389.txt +example_input_file6390.txt +example_input_file6391.txt +example_input_file6392.txt +example_input_file6393.txt +example_input_file6394.txt +example_input_file6395.txt +example_input_file6396.txt +example_input_file6397.txt +example_input_file6398.txt +example_input_file6399.txt +example_input_file6400.txt +example_input_file6401.txt +example_input_file6402.txt +example_input_file6403.txt +example_input_file6404.txt +example_input_file6405.txt +example_input_file6406.txt +example_input_file6407.txt +example_input_file6408.txt +example_input_file6409.txt +example_input_file6410.txt +example_input_file6411.txt +example_input_file6412.txt +example_input_file6413.txt +example_input_file6414.txt +example_input_file6415.txt +example_input_file6416.txt +example_input_file6417.txt +example_input_file6418.txt +example_input_file6419.txt +example_input_file6420.txt +example_input_file6421.txt +example_input_file6422.txt +example_input_file6423.txt +example_input_file6424.txt +example_input_file6425.txt +example_input_file6426.txt +example_input_file6427.txt +example_input_file6428.txt +example_input_file6429.txt +example_input_file6430.txt +example_input_file6431.txt +example_input_file6432.txt +example_input_file6433.txt +example_input_file6434.txt +example_input_file6435.txt +example_input_file6436.txt +example_input_file6437.txt +example_input_file6438.txt +example_input_file6439.txt +example_input_file6440.txt +example_input_file6441.txt +example_input_file6442.txt +example_input_file6443.txt +example_input_file6444.txt +example_input_file6445.txt +example_input_file6446.txt +example_input_file6447.txt +example_input_file6448.txt +example_input_file6449.txt +example_input_file6450.txt +example_input_file6451.txt +example_input_file6452.txt +example_input_file6453.txt +example_input_file6454.txt +example_input_file6455.txt +example_input_file6456.txt +example_input_file6457.txt +example_input_file6458.txt +example_input_file6459.txt +example_input_file6460.txt +example_input_file6461.txt +example_input_file6462.txt +example_input_file6463.txt +example_input_file6464.txt +example_input_file6465.txt +example_input_file6466.txt +example_input_file6467.txt +example_input_file6468.txt +example_input_file6469.txt +example_input_file6470.txt +example_input_file6471.txt +example_input_file6472.txt +example_input_file6473.txt +example_input_file6474.txt +example_input_file6475.txt +example_input_file6476.txt +example_input_file6477.txt +example_input_file6478.txt +example_input_file6479.txt +example_input_file6480.txt +example_input_file6481.txt +example_input_file6482.txt +example_input_file6483.txt +example_input_file6484.txt +example_input_file6485.txt +example_input_file6486.txt +example_input_file6487.txt +example_input_file6488.txt +example_input_file6489.txt +example_input_file6490.txt +example_input_file6491.txt +example_input_file6492.txt +example_input_file6493.txt +example_input_file6494.txt +example_input_file6495.txt +example_input_file6496.txt +example_input_file6497.txt +example_input_file6498.txt +example_input_file6499.txt +example_input_file6500.txt +example_input_file6501.txt +example_input_file6502.txt +example_input_file6503.txt +example_input_file6504.txt +example_input_file6505.txt +example_input_file6506.txt +example_input_file6507.txt +example_input_file6508.txt +example_input_file6509.txt +example_input_file6510.txt +example_input_file6511.txt +example_input_file6512.txt +example_input_file6513.txt +example_input_file6514.txt +example_input_file6515.txt +example_input_file6516.txt +example_input_file6517.txt +example_input_file6518.txt +example_input_file6519.txt +example_input_file6520.txt +example_input_file6521.txt +example_input_file6522.txt +example_input_file6523.txt +example_input_file6524.txt +example_input_file6525.txt +example_input_file6526.txt +example_input_file6527.txt +example_input_file6528.txt +example_input_file6529.txt +example_input_file6530.txt +example_input_file6531.txt +example_input_file6532.txt +example_input_file6533.txt +example_input_file6534.txt +example_input_file6535.txt +example_input_file6536.txt +example_input_file6537.txt +example_input_file6538.txt +example_input_file6539.txt +example_input_file6540.txt +example_input_file6541.txt +example_input_file6542.txt +example_input_file6543.txt +example_input_file6544.txt +example_input_file6545.txt +example_input_file6546.txt +example_input_file6547.txt +example_input_file6548.txt +example_input_file6549.txt +example_input_file6550.txt +example_input_file6551.txt +example_input_file6552.txt +example_input_file6553.txt +example_input_file6554.txt +example_input_file6555.txt +example_input_file6556.txt +example_input_file6557.txt +example_input_file6558.txt +example_input_file6559.txt +example_input_file6560.txt +example_input_file6561.txt +example_input_file6562.txt +example_input_file6563.txt +example_input_file6564.txt +example_input_file6565.txt +example_input_file6566.txt +example_input_file6567.txt +example_input_file6568.txt +example_input_file6569.txt +example_input_file6570.txt +example_input_file6571.txt +example_input_file6572.txt +example_input_file6573.txt +example_input_file6574.txt +example_input_file6575.txt +example_input_file6576.txt +example_input_file6577.txt +example_input_file6578.txt +example_input_file6579.txt +example_input_file6580.txt +example_input_file6581.txt +example_input_file6582.txt +example_input_file6583.txt +example_input_file6584.txt +example_input_file6585.txt +example_input_file6586.txt +example_input_file6587.txt +example_input_file6588.txt +example_input_file6589.txt +example_input_file6590.txt +example_input_file6591.txt +example_input_file6592.txt +example_input_file6593.txt +example_input_file6594.txt +example_input_file6595.txt +example_input_file6596.txt +example_input_file6597.txt +example_input_file6598.txt +example_input_file6599.txt +example_input_file6600.txt +example_input_file6601.txt +example_input_file6602.txt +example_input_file6603.txt +example_input_file6604.txt +example_input_file6605.txt +example_input_file6606.txt +example_input_file6607.txt +example_input_file6608.txt +example_input_file6609.txt +example_input_file6610.txt +example_input_file6611.txt +example_input_file6612.txt +example_input_file6613.txt +example_input_file6614.txt +example_input_file6615.txt +example_input_file6616.txt +example_input_file6617.txt +example_input_file6618.txt +example_input_file6619.txt +example_input_file6620.txt +example_input_file6621.txt +example_input_file6622.txt +example_input_file6623.txt +example_input_file6624.txt +example_input_file6625.txt +example_input_file6626.txt +example_input_file6627.txt +example_input_file6628.txt +example_input_file6629.txt +example_input_file6630.txt +example_input_file6631.txt +example_input_file6632.txt +example_input_file6633.txt +example_input_file6634.txt +example_input_file6635.txt +example_input_file6636.txt +example_input_file6637.txt +example_input_file6638.txt +example_input_file6639.txt +example_input_file6640.txt +example_input_file6641.txt +example_input_file6642.txt +example_input_file6643.txt +example_input_file6644.txt +example_input_file6645.txt +example_input_file6646.txt +example_input_file6647.txt +example_input_file6648.txt +example_input_file6649.txt +example_input_file6650.txt +example_input_file6651.txt +example_input_file6652.txt +example_input_file6653.txt +example_input_file6654.txt +example_input_file6655.txt +example_input_file6656.txt +example_input_file6657.txt +example_input_file6658.txt +example_input_file6659.txt +example_input_file6660.txt +example_input_file6661.txt +example_input_file6662.txt +example_input_file6663.txt +example_input_file6664.txt +example_input_file6665.txt +example_input_file6666.txt +example_input_file6667.txt +example_input_file6668.txt +example_input_file6669.txt +example_input_file6670.txt +example_input_file6671.txt +example_input_file6672.txt +example_input_file6673.txt +example_input_file6674.txt +example_input_file6675.txt +example_input_file6676.txt +example_input_file6677.txt +example_input_file6678.txt +example_input_file6679.txt +example_input_file6680.txt +example_input_file6681.txt +example_input_file6682.txt +example_input_file6683.txt +example_input_file6684.txt +example_input_file6685.txt +example_input_file6686.txt +example_input_file6687.txt +example_input_file6688.txt +example_input_file6689.txt +example_input_file6690.txt +example_input_file6691.txt +example_input_file6692.txt +example_input_file6693.txt +example_input_file6694.txt +example_input_file6695.txt +example_input_file6696.txt +example_input_file6697.txt +example_input_file6698.txt +example_input_file6699.txt +example_input_file6700.txt +example_input_file6701.txt +example_input_file6702.txt +example_input_file6703.txt +example_input_file6704.txt +example_input_file6705.txt +example_input_file6706.txt +example_input_file6707.txt +example_input_file6708.txt +example_input_file6709.txt +example_input_file6710.txt +example_input_file6711.txt +example_input_file6712.txt +example_input_file6713.txt +example_input_file6714.txt +example_input_file6715.txt +example_input_file6716.txt +example_input_file6717.txt +example_input_file6718.txt +example_input_file6719.txt +example_input_file6720.txt +example_input_file6721.txt +example_input_file6722.txt +example_input_file6723.txt +example_input_file6724.txt +example_input_file6725.txt +example_input_file6726.txt +example_input_file6727.txt +example_input_file6728.txt +example_input_file6729.txt +example_input_file6730.txt +example_input_file6731.txt +example_input_file6732.txt +example_input_file6733.txt +example_input_file6734.txt +example_input_file6735.txt +example_input_file6736.txt +example_input_file6737.txt +example_input_file6738.txt +example_input_file6739.txt +example_input_file6740.txt +example_input_file6741.txt +example_input_file6742.txt +example_input_file6743.txt +example_input_file6744.txt +example_input_file6745.txt +example_input_file6746.txt +example_input_file6747.txt +example_input_file6748.txt +example_input_file6749.txt +example_input_file6750.txt +example_input_file6751.txt +example_input_file6752.txt +example_input_file6753.txt +example_input_file6754.txt +example_input_file6755.txt +example_input_file6756.txt +example_input_file6757.txt +example_input_file6758.txt +example_input_file6759.txt +example_input_file6760.txt +example_input_file6761.txt +example_input_file6762.txt +example_input_file6763.txt +example_input_file6764.txt +example_input_file6765.txt +example_input_file6766.txt +example_input_file6767.txt +example_input_file6768.txt +example_input_file6769.txt +example_input_file6770.txt +example_input_file6771.txt +example_input_file6772.txt +example_input_file6773.txt +example_input_file6774.txt +example_input_file6775.txt +example_input_file6776.txt +example_input_file6777.txt +example_input_file6778.txt +example_input_file6779.txt +example_input_file6780.txt +example_input_file6781.txt +example_input_file6782.txt +example_input_file6783.txt +example_input_file6784.txt +example_input_file6785.txt +example_input_file6786.txt +example_input_file6787.txt +example_input_file6788.txt +example_input_file6789.txt +example_input_file6790.txt +example_input_file6791.txt +example_input_file6792.txt +example_input_file6793.txt +example_input_file6794.txt +example_input_file6795.txt +example_input_file6796.txt +example_input_file6797.txt +example_input_file6798.txt +example_input_file6799.txt +example_input_file6800.txt +example_input_file6801.txt +example_input_file6802.txt +example_input_file6803.txt +example_input_file6804.txt +example_input_file6805.txt +example_input_file6806.txt +example_input_file6807.txt +example_input_file6808.txt +example_input_file6809.txt +example_input_file6810.txt +example_input_file6811.txt +example_input_file6812.txt +example_input_file6813.txt +example_input_file6814.txt +example_input_file6815.txt +example_input_file6816.txt +example_input_file6817.txt +example_input_file6818.txt +example_input_file6819.txt +example_input_file6820.txt +example_input_file6821.txt +example_input_file6822.txt +example_input_file6823.txt +example_input_file6824.txt +example_input_file6825.txt +example_input_file6826.txt +example_input_file6827.txt +example_input_file6828.txt +example_input_file6829.txt +example_input_file6830.txt +example_input_file6831.txt +example_input_file6832.txt +example_input_file6833.txt +example_input_file6834.txt +example_input_file6835.txt +example_input_file6836.txt +example_input_file6837.txt +example_input_file6838.txt +example_input_file6839.txt +example_input_file6840.txt +example_input_file6841.txt +example_input_file6842.txt +example_input_file6843.txt +example_input_file6844.txt +example_input_file6845.txt +example_input_file6846.txt +example_input_file6847.txt +example_input_file6848.txt +example_input_file6849.txt +example_input_file6850.txt +example_input_file6851.txt +example_input_file6852.txt +example_input_file6853.txt +example_input_file6854.txt +example_input_file6855.txt +example_input_file6856.txt +example_input_file6857.txt +example_input_file6858.txt +example_input_file6859.txt +example_input_file6860.txt +example_input_file6861.txt +example_input_file6862.txt +example_input_file6863.txt +example_input_file6864.txt +example_input_file6865.txt +example_input_file6866.txt +example_input_file6867.txt +example_input_file6868.txt +example_input_file6869.txt +example_input_file6870.txt +example_input_file6871.txt +example_input_file6872.txt +example_input_file6873.txt +example_input_file6874.txt +example_input_file6875.txt +example_input_file6876.txt +example_input_file6877.txt +example_input_file6878.txt +example_input_file6879.txt +example_input_file6880.txt +example_input_file6881.txt +example_input_file6882.txt +example_input_file6883.txt +example_input_file6884.txt +example_input_file6885.txt +example_input_file6886.txt +example_input_file6887.txt +example_input_file6888.txt +example_input_file6889.txt +example_input_file6890.txt +example_input_file6891.txt +example_input_file6892.txt +example_input_file6893.txt +example_input_file6894.txt +example_input_file6895.txt +example_input_file6896.txt +example_input_file6897.txt +example_input_file6898.txt +example_input_file6899.txt +example_input_file6900.txt +example_input_file6901.txt +example_input_file6902.txt +example_input_file6903.txt +example_input_file6904.txt +example_input_file6905.txt +example_input_file6906.txt +example_input_file6907.txt +example_input_file6908.txt +example_input_file6909.txt +example_input_file6910.txt +example_input_file6911.txt +example_input_file6912.txt +example_input_file6913.txt +example_input_file6914.txt +example_input_file6915.txt +example_input_file6916.txt +example_input_file6917.txt +example_input_file6918.txt +example_input_file6919.txt +example_input_file6920.txt +example_input_file6921.txt +example_input_file6922.txt +example_input_file6923.txt +example_input_file6924.txt +example_input_file6925.txt +example_input_file6926.txt +example_input_file6927.txt +example_input_file6928.txt +example_input_file6929.txt +example_input_file6930.txt +example_input_file6931.txt +example_input_file6932.txt +example_input_file6933.txt +example_input_file6934.txt +example_input_file6935.txt +example_input_file6936.txt +example_input_file6937.txt +example_input_file6938.txt +example_input_file6939.txt +example_input_file6940.txt +example_input_file6941.txt +example_input_file6942.txt +example_input_file6943.txt +example_input_file6944.txt +example_input_file6945.txt +example_input_file6946.txt +example_input_file6947.txt +example_input_file6948.txt +example_input_file6949.txt +example_input_file6950.txt +example_input_file6951.txt +example_input_file6952.txt +example_input_file6953.txt +example_input_file6954.txt +example_input_file6955.txt +example_input_file6956.txt +example_input_file6957.txt +example_input_file6958.txt +example_input_file6959.txt +example_input_file6960.txt +example_input_file6961.txt +example_input_file6962.txt +example_input_file6963.txt +example_input_file6964.txt +example_input_file6965.txt +example_input_file6966.txt +example_input_file6967.txt +example_input_file6968.txt +example_input_file6969.txt +example_input_file6970.txt +example_input_file6971.txt +example_input_file6972.txt +example_input_file6973.txt +example_input_file6974.txt +example_input_file6975.txt +example_input_file6976.txt +example_input_file6977.txt +example_input_file6978.txt +example_input_file6979.txt +example_input_file6980.txt +example_input_file6981.txt +example_input_file6982.txt +example_input_file6983.txt +example_input_file6984.txt +example_input_file6985.txt +example_input_file6986.txt +example_input_file6987.txt +example_input_file6988.txt +example_input_file6989.txt +example_input_file6990.txt +example_input_file6991.txt +example_input_file6992.txt +example_input_file6993.txt +example_input_file6994.txt +example_input_file6995.txt +example_input_file6996.txt +example_input_file6997.txt +example_input_file6998.txt +example_input_file6999.txt +example_input_file7000.txt +example_input_file7001.txt +example_input_file7002.txt +example_input_file7003.txt +example_input_file7004.txt +example_input_file7005.txt +example_input_file7006.txt +example_input_file7007.txt +example_input_file7008.txt +example_input_file7009.txt +example_input_file7010.txt +example_input_file7011.txt +example_input_file7012.txt +example_input_file7013.txt +example_input_file7014.txt +example_input_file7015.txt +example_input_file7016.txt +example_input_file7017.txt +example_input_file7018.txt +example_input_file7019.txt +example_input_file7020.txt +example_input_file7021.txt +example_input_file7022.txt +example_input_file7023.txt +example_input_file7024.txt +example_input_file7025.txt +example_input_file7026.txt +example_input_file7027.txt +example_input_file7028.txt +example_input_file7029.txt +example_input_file7030.txt +example_input_file7031.txt +example_input_file7032.txt +example_input_file7033.txt +example_input_file7034.txt +example_input_file7035.txt +example_input_file7036.txt +example_input_file7037.txt +example_input_file7038.txt +example_input_file7039.txt +example_input_file7040.txt +example_input_file7041.txt +example_input_file7042.txt +example_input_file7043.txt +example_input_file7044.txt +example_input_file7045.txt +example_input_file7046.txt +example_input_file7047.txt +example_input_file7048.txt +example_input_file7049.txt +example_input_file7050.txt +example_input_file7051.txt +example_input_file7052.txt +example_input_file7053.txt +example_input_file7054.txt +example_input_file7055.txt +example_input_file7056.txt +example_input_file7057.txt +example_input_file7058.txt +example_input_file7059.txt +example_input_file7060.txt +example_input_file7061.txt +example_input_file7062.txt +example_input_file7063.txt +example_input_file7064.txt +example_input_file7065.txt +example_input_file7066.txt +example_input_file7067.txt +example_input_file7068.txt +example_input_file7069.txt +example_input_file7070.txt +example_input_file7071.txt +example_input_file7072.txt +example_input_file7073.txt +example_input_file7074.txt +example_input_file7075.txt +example_input_file7076.txt +example_input_file7077.txt +example_input_file7078.txt +example_input_file7079.txt +example_input_file7080.txt +example_input_file7081.txt +example_input_file7082.txt +example_input_file7083.txt +example_input_file7084.txt +example_input_file7085.txt +example_input_file7086.txt +example_input_file7087.txt +example_input_file7088.txt +example_input_file7089.txt +example_input_file7090.txt +example_input_file7091.txt +example_input_file7092.txt +example_input_file7093.txt +example_input_file7094.txt +example_input_file7095.txt +example_input_file7096.txt +example_input_file7097.txt +example_input_file7098.txt +example_input_file7099.txt +example_input_file7100.txt +example_input_file7101.txt +example_input_file7102.txt +example_input_file7103.txt +example_input_file7104.txt +example_input_file7105.txt +example_input_file7106.txt +example_input_file7107.txt +example_input_file7108.txt +example_input_file7109.txt +example_input_file7110.txt +example_input_file7111.txt +example_input_file7112.txt +example_input_file7113.txt +example_input_file7114.txt +example_input_file7115.txt +example_input_file7116.txt +example_input_file7117.txt +example_input_file7118.txt +example_input_file7119.txt +example_input_file7120.txt +example_input_file7121.txt +example_input_file7122.txt +example_input_file7123.txt +example_input_file7124.txt +example_input_file7125.txt +example_input_file7126.txt +example_input_file7127.txt +example_input_file7128.txt +example_input_file7129.txt +example_input_file7130.txt +example_input_file7131.txt +example_input_file7132.txt +example_input_file7133.txt +example_input_file7134.txt +example_input_file7135.txt +example_input_file7136.txt +example_input_file7137.txt +example_input_file7138.txt +example_input_file7139.txt +example_input_file7140.txt +example_input_file7141.txt +example_input_file7142.txt +example_input_file7143.txt +example_input_file7144.txt +example_input_file7145.txt +example_input_file7146.txt +example_input_file7147.txt +example_input_file7148.txt +example_input_file7149.txt +example_input_file7150.txt +example_input_file7151.txt +example_input_file7152.txt +example_input_file7153.txt +example_input_file7154.txt +example_input_file7155.txt +example_input_file7156.txt +example_input_file7157.txt +example_input_file7158.txt +example_input_file7159.txt +example_input_file7160.txt +example_input_file7161.txt +example_input_file7162.txt +example_input_file7163.txt +example_input_file7164.txt +example_input_file7165.txt +example_input_file7166.txt +example_input_file7167.txt +example_input_file7168.txt +example_input_file7169.txt +example_input_file7170.txt +example_input_file7171.txt +example_input_file7172.txt +example_input_file7173.txt +example_input_file7174.txt +example_input_file7175.txt +example_input_file7176.txt +example_input_file7177.txt +example_input_file7178.txt +example_input_file7179.txt +example_input_file7180.txt +example_input_file7181.txt +example_input_file7182.txt +example_input_file7183.txt +example_input_file7184.txt +example_input_file7185.txt +example_input_file7186.txt +example_input_file7187.txt +example_input_file7188.txt +example_input_file7189.txt +example_input_file7190.txt +example_input_file7191.txt +example_input_file7192.txt +example_input_file7193.txt +example_input_file7194.txt +example_input_file7195.txt +example_input_file7196.txt +example_input_file7197.txt +example_input_file7198.txt +example_input_file7199.txt +example_input_file7200.txt +example_input_file7201.txt +example_input_file7202.txt +example_input_file7203.txt +example_input_file7204.txt +example_input_file7205.txt +example_input_file7206.txt +example_input_file7207.txt +example_input_file7208.txt +example_input_file7209.txt +example_input_file7210.txt +example_input_file7211.txt +example_input_file7212.txt +example_input_file7213.txt +example_input_file7214.txt +example_input_file7215.txt +example_input_file7216.txt +example_input_file7217.txt +example_input_file7218.txt +example_input_file7219.txt +example_input_file7220.txt +example_input_file7221.txt +example_input_file7222.txt +example_input_file7223.txt +example_input_file7224.txt +example_input_file7225.txt +example_input_file7226.txt +example_input_file7227.txt +example_input_file7228.txt +example_input_file7229.txt +example_input_file7230.txt +example_input_file7231.txt +example_input_file7232.txt +example_input_file7233.txt +example_input_file7234.txt +example_input_file7235.txt +example_input_file7236.txt +example_input_file7237.txt +example_input_file7238.txt +example_input_file7239.txt +example_input_file7240.txt +example_input_file7241.txt +example_input_file7242.txt +example_input_file7243.txt +example_input_file7244.txt +example_input_file7245.txt +example_input_file7246.txt +example_input_file7247.txt +example_input_file7248.txt +example_input_file7249.txt +example_input_file7250.txt +example_input_file7251.txt +example_input_file7252.txt +example_input_file7253.txt +example_input_file7254.txt +example_input_file7255.txt +example_input_file7256.txt +example_input_file7257.txt +example_input_file7258.txt +example_input_file7259.txt +example_input_file7260.txt +example_input_file7261.txt +example_input_file7262.txt +example_input_file7263.txt +example_input_file7264.txt +example_input_file7265.txt +example_input_file7266.txt +example_input_file7267.txt +example_input_file7268.txt +example_input_file7269.txt +example_input_file7270.txt +example_input_file7271.txt +example_input_file7272.txt +example_input_file7273.txt +example_input_file7274.txt +example_input_file7275.txt +example_input_file7276.txt +example_input_file7277.txt +example_input_file7278.txt +example_input_file7279.txt +example_input_file7280.txt +example_input_file7281.txt +example_input_file7282.txt +example_input_file7283.txt +example_input_file7284.txt +example_input_file7285.txt +example_input_file7286.txt +example_input_file7287.txt +example_input_file7288.txt +example_input_file7289.txt +example_input_file7290.txt +example_input_file7291.txt +example_input_file7292.txt +example_input_file7293.txt +example_input_file7294.txt +example_input_file7295.txt +example_input_file7296.txt +example_input_file7297.txt +example_input_file7298.txt +example_input_file7299.txt +example_input_file7300.txt +example_input_file7301.txt +example_input_file7302.txt +example_input_file7303.txt +example_input_file7304.txt +example_input_file7305.txt +example_input_file7306.txt +example_input_file7307.txt +example_input_file7308.txt +example_input_file7309.txt +example_input_file7310.txt +example_input_file7311.txt +example_input_file7312.txt +example_input_file7313.txt +example_input_file7314.txt +example_input_file7315.txt +example_input_file7316.txt +example_input_file7317.txt +example_input_file7318.txt +example_input_file7319.txt +example_input_file7320.txt +example_input_file7321.txt +example_input_file7322.txt +example_input_file7323.txt +example_input_file7324.txt +example_input_file7325.txt +example_input_file7326.txt +example_input_file7327.txt +example_input_file7328.txt +example_input_file7329.txt +example_input_file7330.txt +example_input_file7331.txt +example_input_file7332.txt +example_input_file7333.txt +example_input_file7334.txt +example_input_file7335.txt +example_input_file7336.txt +example_input_file7337.txt +example_input_file7338.txt +example_input_file7339.txt +example_input_file7340.txt +example_input_file7341.txt +example_input_file7342.txt +example_input_file7343.txt +example_input_file7344.txt +example_input_file7345.txt +example_input_file7346.txt +example_input_file7347.txt +example_input_file7348.txt +example_input_file7349.txt +example_input_file7350.txt +example_input_file7351.txt +example_input_file7352.txt +example_input_file7353.txt +example_input_file7354.txt +example_input_file7355.txt +example_input_file7356.txt +example_input_file7357.txt +example_input_file7358.txt +example_input_file7359.txt +example_input_file7360.txt +example_input_file7361.txt +example_input_file7362.txt +example_input_file7363.txt +example_input_file7364.txt +example_input_file7365.txt +example_input_file7366.txt +example_input_file7367.txt +example_input_file7368.txt +example_input_file7369.txt +example_input_file7370.txt +example_input_file7371.txt +example_input_file7372.txt +example_input_file7373.txt +example_input_file7374.txt +example_input_file7375.txt +example_input_file7376.txt +example_input_file7377.txt +example_input_file7378.txt +example_input_file7379.txt +example_input_file7380.txt +example_input_file7381.txt +example_input_file7382.txt +example_input_file7383.txt +example_input_file7384.txt +example_input_file7385.txt +example_input_file7386.txt +example_input_file7387.txt +example_input_file7388.txt +example_input_file7389.txt +example_input_file7390.txt +example_input_file7391.txt +example_input_file7392.txt +example_input_file7393.txt +example_input_file7394.txt +example_input_file7395.txt +example_input_file7396.txt +example_input_file7397.txt +example_input_file7398.txt +example_input_file7399.txt +example_input_file7400.txt +example_input_file7401.txt +example_input_file7402.txt +example_input_file7403.txt +example_input_file7404.txt +example_input_file7405.txt +example_input_file7406.txt +example_input_file7407.txt +example_input_file7408.txt +example_input_file7409.txt +example_input_file7410.txt +example_input_file7411.txt +example_input_file7412.txt +example_input_file7413.txt +example_input_file7414.txt +example_input_file7415.txt +example_input_file7416.txt +example_input_file7417.txt +example_input_file7418.txt +example_input_file7419.txt +example_input_file7420.txt +example_input_file7421.txt +example_input_file7422.txt +example_input_file7423.txt +example_input_file7424.txt +example_input_file7425.txt +example_input_file7426.txt +example_input_file7427.txt +example_input_file7428.txt +example_input_file7429.txt +example_input_file7430.txt +example_input_file7431.txt +example_input_file7432.txt +example_input_file7433.txt +example_input_file7434.txt +example_input_file7435.txt +example_input_file7436.txt +example_input_file7437.txt +example_input_file7438.txt +example_input_file7439.txt +example_input_file7440.txt +example_input_file7441.txt +example_input_file7442.txt +example_input_file7443.txt +example_input_file7444.txt +example_input_file7445.txt +example_input_file7446.txt +example_input_file7447.txt +example_input_file7448.txt +example_input_file7449.txt +example_input_file7450.txt +example_input_file7451.txt +example_input_file7452.txt +example_input_file7453.txt +example_input_file7454.txt +example_input_file7455.txt +example_input_file7456.txt +example_input_file7457.txt +example_input_file7458.txt +example_input_file7459.txt +example_input_file7460.txt +example_input_file7461.txt +example_input_file7462.txt +example_input_file7463.txt +example_input_file7464.txt +example_input_file7465.txt +example_input_file7466.txt +example_input_file7467.txt +example_input_file7468.txt +example_input_file7469.txt +example_input_file7470.txt +example_input_file7471.txt +example_input_file7472.txt +example_input_file7473.txt +example_input_file7474.txt +example_input_file7475.txt +example_input_file7476.txt +example_input_file7477.txt +example_input_file7478.txt +example_input_file7479.txt +example_input_file7480.txt +example_input_file7481.txt +example_input_file7482.txt +example_input_file7483.txt +example_input_file7484.txt +example_input_file7485.txt +example_input_file7486.txt +example_input_file7487.txt +example_input_file7488.txt +example_input_file7489.txt +example_input_file7490.txt +example_input_file7491.txt +example_input_file7492.txt +example_input_file7493.txt +example_input_file7494.txt +example_input_file7495.txt +example_input_file7496.txt +example_input_file7497.txt +example_input_file7498.txt +example_input_file7499.txt +example_input_file7500.txt +example_input_file7501.txt +example_input_file7502.txt +example_input_file7503.txt +example_input_file7504.txt +example_input_file7505.txt +example_input_file7506.txt +example_input_file7507.txt +example_input_file7508.txt +example_input_file7509.txt +example_input_file7510.txt +example_input_file7511.txt +example_input_file7512.txt +example_input_file7513.txt +example_input_file7514.txt +example_input_file7515.txt +example_input_file7516.txt +example_input_file7517.txt +example_input_file7518.txt +example_input_file7519.txt +example_input_file7520.txt +example_input_file7521.txt +example_input_file7522.txt +example_input_file7523.txt +example_input_file7524.txt +example_input_file7525.txt +example_input_file7526.txt +example_input_file7527.txt +example_input_file7528.txt +example_input_file7529.txt +example_input_file7530.txt +example_input_file7531.txt +example_input_file7532.txt +example_input_file7533.txt +example_input_file7534.txt +example_input_file7535.txt +example_input_file7536.txt +example_input_file7537.txt +example_input_file7538.txt +example_input_file7539.txt +example_input_file7540.txt +example_input_file7541.txt +example_input_file7542.txt +example_input_file7543.txt +example_input_file7544.txt +example_input_file7545.txt +example_input_file7546.txt +example_input_file7547.txt +example_input_file7548.txt +example_input_file7549.txt +example_input_file7550.txt +example_input_file7551.txt +example_input_file7552.txt +example_input_file7553.txt +example_input_file7554.txt +example_input_file7555.txt +example_input_file7556.txt +example_input_file7557.txt +example_input_file7558.txt +example_input_file7559.txt +example_input_file7560.txt +example_input_file7561.txt +example_input_file7562.txt +example_input_file7563.txt +example_input_file7564.txt +example_input_file7565.txt +example_input_file7566.txt +example_input_file7567.txt +example_input_file7568.txt +example_input_file7569.txt +example_input_file7570.txt +example_input_file7571.txt +example_input_file7572.txt +example_input_file7573.txt +example_input_file7574.txt +example_input_file7575.txt +example_input_file7576.txt +example_input_file7577.txt +example_input_file7578.txt +example_input_file7579.txt +example_input_file7580.txt +example_input_file7581.txt +example_input_file7582.txt +example_input_file7583.txt +example_input_file7584.txt +example_input_file7585.txt +example_input_file7586.txt +example_input_file7587.txt +example_input_file7588.txt +example_input_file7589.txt +example_input_file7590.txt +example_input_file7591.txt +example_input_file7592.txt +example_input_file7593.txt +example_input_file7594.txt +example_input_file7595.txt +example_input_file7596.txt +example_input_file7597.txt +example_input_file7598.txt +example_input_file7599.txt +example_input_file7600.txt +example_input_file7601.txt +example_input_file7602.txt +example_input_file7603.txt +example_input_file7604.txt +example_input_file7605.txt +example_input_file7606.txt +example_input_file7607.txt +example_input_file7608.txt +example_input_file7609.txt +example_input_file7610.txt +example_input_file7611.txt +example_input_file7612.txt +example_input_file7613.txt +example_input_file7614.txt +example_input_file7615.txt +example_input_file7616.txt +example_input_file7617.txt +example_input_file7618.txt +example_input_file7619.txt +example_input_file7620.txt +example_input_file7621.txt +example_input_file7622.txt +example_input_file7623.txt +example_input_file7624.txt +example_input_file7625.txt +example_input_file7626.txt +example_input_file7627.txt +example_input_file7628.txt +example_input_file7629.txt +example_input_file7630.txt +example_input_file7631.txt +example_input_file7632.txt +example_input_file7633.txt +example_input_file7634.txt +example_input_file7635.txt +example_input_file7636.txt +example_input_file7637.txt +example_input_file7638.txt +example_input_file7639.txt +example_input_file7640.txt +example_input_file7641.txt +example_input_file7642.txt +example_input_file7643.txt +example_input_file7644.txt +example_input_file7645.txt +example_input_file7646.txt +example_input_file7647.txt +example_input_file7648.txt +example_input_file7649.txt +example_input_file7650.txt +example_input_file7651.txt +example_input_file7652.txt +example_input_file7653.txt +example_input_file7654.txt +example_input_file7655.txt +example_input_file7656.txt +example_input_file7657.txt +example_input_file7658.txt +example_input_file7659.txt +example_input_file7660.txt +example_input_file7661.txt +example_input_file7662.txt +example_input_file7663.txt +example_input_file7664.txt +example_input_file7665.txt +example_input_file7666.txt +example_input_file7667.txt +example_input_file7668.txt +example_input_file7669.txt +example_input_file7670.txt +example_input_file7671.txt +example_input_file7672.txt +example_input_file7673.txt +example_input_file7674.txt +example_input_file7675.txt +example_input_file7676.txt +example_input_file7677.txt +example_input_file7678.txt +example_input_file7679.txt +example_input_file7680.txt +example_input_file7681.txt +example_input_file7682.txt +example_input_file7683.txt +example_input_file7684.txt +example_input_file7685.txt +example_input_file7686.txt +example_input_file7687.txt +example_input_file7688.txt +example_input_file7689.txt +example_input_file7690.txt +example_input_file7691.txt +example_input_file7692.txt +example_input_file7693.txt +example_input_file7694.txt +example_input_file7695.txt +example_input_file7696.txt +example_input_file7697.txt +example_input_file7698.txt +example_input_file7699.txt +example_input_file7700.txt +example_input_file7701.txt +example_input_file7702.txt +example_input_file7703.txt +example_input_file7704.txt +example_input_file7705.txt +example_input_file7706.txt +example_input_file7707.txt +example_input_file7708.txt +example_input_file7709.txt +example_input_file7710.txt +example_input_file7711.txt +example_input_file7712.txt +example_input_file7713.txt +example_input_file7714.txt +example_input_file7715.txt +example_input_file7716.txt +example_input_file7717.txt +example_input_file7718.txt +example_input_file7719.txt +example_input_file7720.txt +example_input_file7721.txt +example_input_file7722.txt +example_input_file7723.txt +example_input_file7724.txt +example_input_file7725.txt +example_input_file7726.txt +example_input_file7727.txt +example_input_file7728.txt +example_input_file7729.txt +example_input_file7730.txt +example_input_file7731.txt +example_input_file7732.txt +example_input_file7733.txt +example_input_file7734.txt +example_input_file7735.txt +example_input_file7736.txt +example_input_file7737.txt +example_input_file7738.txt +example_input_file7739.txt +example_input_file7740.txt +example_input_file7741.txt +example_input_file7742.txt +example_input_file7743.txt +example_input_file7744.txt +example_input_file7745.txt +example_input_file7746.txt +example_input_file7747.txt +example_input_file7748.txt +example_input_file7749.txt +example_input_file7750.txt +example_input_file7751.txt +example_input_file7752.txt +example_input_file7753.txt +example_input_file7754.txt +example_input_file7755.txt +example_input_file7756.txt +example_input_file7757.txt +example_input_file7758.txt +example_input_file7759.txt +example_input_file7760.txt +example_input_file7761.txt +example_input_file7762.txt +example_input_file7763.txt +example_input_file7764.txt +example_input_file7765.txt +example_input_file7766.txt +example_input_file7767.txt +example_input_file7768.txt +example_input_file7769.txt +example_input_file7770.txt +example_input_file7771.txt +example_input_file7772.txt +example_input_file7773.txt +example_input_file7774.txt +example_input_file7775.txt +example_input_file7776.txt +example_input_file7777.txt +example_input_file7778.txt +example_input_file7779.txt +example_input_file7780.txt +example_input_file7781.txt +example_input_file7782.txt +example_input_file7783.txt +example_input_file7784.txt +example_input_file7785.txt +example_input_file7786.txt +example_input_file7787.txt +example_input_file7788.txt +example_input_file7789.txt +example_input_file7790.txt +example_input_file7791.txt +example_input_file7792.txt +example_input_file7793.txt +example_input_file7794.txt +example_input_file7795.txt +example_input_file7796.txt +example_input_file7797.txt +example_input_file7798.txt +example_input_file7799.txt +example_input_file7800.txt +example_input_file7801.txt +example_input_file7802.txt +example_input_file7803.txt +example_input_file7804.txt +example_input_file7805.txt +example_input_file7806.txt +example_input_file7807.txt +example_input_file7808.txt +example_input_file7809.txt +example_input_file7810.txt +example_input_file7811.txt +example_input_file7812.txt +example_input_file7813.txt +example_input_file7814.txt +example_input_file7815.txt +example_input_file7816.txt +example_input_file7817.txt +example_input_file7818.txt +example_input_file7819.txt +example_input_file7820.txt +example_input_file7821.txt +example_input_file7822.txt +example_input_file7823.txt +example_input_file7824.txt +example_input_file7825.txt +example_input_file7826.txt +example_input_file7827.txt +example_input_file7828.txt +example_input_file7829.txt +example_input_file7830.txt +example_input_file7831.txt +example_input_file7832.txt +example_input_file7833.txt +example_input_file7834.txt +example_input_file7835.txt +example_input_file7836.txt +example_input_file7837.txt +example_input_file7838.txt +example_input_file7839.txt +example_input_file7840.txt +example_input_file7841.txt +example_input_file7842.txt +example_input_file7843.txt +example_input_file7844.txt +example_input_file7845.txt +example_input_file7846.txt +example_input_file7847.txt +example_input_file7848.txt +example_input_file7849.txt +example_input_file7850.txt +example_input_file7851.txt +example_input_file7852.txt +example_input_file7853.txt +example_input_file7854.txt +example_input_file7855.txt +example_input_file7856.txt +example_input_file7857.txt +example_input_file7858.txt +example_input_file7859.txt +example_input_file7860.txt +example_input_file7861.txt +example_input_file7862.txt +example_input_file7863.txt +example_input_file7864.txt +example_input_file7865.txt +example_input_file7866.txt +example_input_file7867.txt +example_input_file7868.txt +example_input_file7869.txt +example_input_file7870.txt +example_input_file7871.txt +example_input_file7872.txt +example_input_file7873.txt +example_input_file7874.txt +example_input_file7875.txt +example_input_file7876.txt +example_input_file7877.txt +example_input_file7878.txt +example_input_file7879.txt +example_input_file7880.txt +example_input_file7881.txt +example_input_file7882.txt +example_input_file7883.txt +example_input_file7884.txt +example_input_file7885.txt +example_input_file7886.txt +example_input_file7887.txt +example_input_file7888.txt +example_input_file7889.txt +example_input_file7890.txt +example_input_file7891.txt +example_input_file7892.txt +example_input_file7893.txt +example_input_file7894.txt +example_input_file7895.txt +example_input_file7896.txt +example_input_file7897.txt +example_input_file7898.txt +example_input_file7899.txt +example_input_file7900.txt +example_input_file7901.txt +example_input_file7902.txt +example_input_file7903.txt +example_input_file7904.txt +example_input_file7905.txt +example_input_file7906.txt +example_input_file7907.txt +example_input_file7908.txt +example_input_file7909.txt +example_input_file7910.txt +example_input_file7911.txt +example_input_file7912.txt +example_input_file7913.txt +example_input_file7914.txt +example_input_file7915.txt +example_input_file7916.txt +example_input_file7917.txt +example_input_file7918.txt +example_input_file7919.txt +example_input_file7920.txt +example_input_file7921.txt +example_input_file7922.txt +example_input_file7923.txt +example_input_file7924.txt +example_input_file7925.txt +example_input_file7926.txt +example_input_file7927.txt +example_input_file7928.txt +example_input_file7929.txt +example_input_file7930.txt +example_input_file7931.txt +example_input_file7932.txt +example_input_file7933.txt +example_input_file7934.txt +example_input_file7935.txt +example_input_file7936.txt +example_input_file7937.txt +example_input_file7938.txt +example_input_file7939.txt +example_input_file7940.txt +example_input_file7941.txt +example_input_file7942.txt +example_input_file7943.txt +example_input_file7944.txt +example_input_file7945.txt +example_input_file7946.txt +example_input_file7947.txt +example_input_file7948.txt +example_input_file7949.txt +example_input_file7950.txt +example_input_file7951.txt +example_input_file7952.txt +example_input_file7953.txt +example_input_file7954.txt +example_input_file7955.txt +example_input_file7956.txt +example_input_file7957.txt +example_input_file7958.txt +example_input_file7959.txt +example_input_file7960.txt +example_input_file7961.txt +example_input_file7962.txt +example_input_file7963.txt +example_input_file7964.txt +example_input_file7965.txt +example_input_file7966.txt +example_input_file7967.txt +example_input_file7968.txt +example_input_file7969.txt +example_input_file7970.txt +example_input_file7971.txt +example_input_file7972.txt +example_input_file7973.txt +example_input_file7974.txt +example_input_file7975.txt +example_input_file7976.txt +example_input_file7977.txt +example_input_file7978.txt +example_input_file7979.txt +example_input_file7980.txt +example_input_file7981.txt +example_input_file7982.txt +example_input_file7983.txt +example_input_file7984.txt +example_input_file7985.txt +example_input_file7986.txt +example_input_file7987.txt +example_input_file7988.txt +example_input_file7989.txt +example_input_file7990.txt +example_input_file7991.txt +example_input_file7992.txt +example_input_file7993.txt +example_input_file7994.txt +example_input_file7995.txt +example_input_file7996.txt +example_input_file7997.txt +example_input_file7998.txt +example_input_file7999.txt +example_input_file8000.txt +example_input_file8001.txt +example_input_file8002.txt +example_input_file8003.txt +example_input_file8004.txt +example_input_file8005.txt +example_input_file8006.txt +example_input_file8007.txt +example_input_file8008.txt +example_input_file8009.txt +example_input_file8010.txt +example_input_file8011.txt +example_input_file8012.txt +example_input_file8013.txt +example_input_file8014.txt +example_input_file8015.txt +example_input_file8016.txt +example_input_file8017.txt +example_input_file8018.txt +example_input_file8019.txt +example_input_file8020.txt +example_input_file8021.txt +example_input_file8022.txt +example_input_file8023.txt +example_input_file8024.txt +example_input_file8025.txt +example_input_file8026.txt +example_input_file8027.txt +example_input_file8028.txt +example_input_file8029.txt +example_input_file8030.txt +example_input_file8031.txt +example_input_file8032.txt +example_input_file8033.txt +example_input_file8034.txt +example_input_file8035.txt +example_input_file8036.txt +example_input_file8037.txt +example_input_file8038.txt +example_input_file8039.txt +example_input_file8040.txt +example_input_file8041.txt +example_input_file8042.txt +example_input_file8043.txt +example_input_file8044.txt +example_input_file8045.txt +example_input_file8046.txt +example_input_file8047.txt +example_input_file8048.txt +example_input_file8049.txt +example_input_file8050.txt +example_input_file8051.txt +example_input_file8052.txt +example_input_file8053.txt +example_input_file8054.txt +example_input_file8055.txt +example_input_file8056.txt +example_input_file8057.txt +example_input_file8058.txt +example_input_file8059.txt +example_input_file8060.txt +example_input_file8061.txt +example_input_file8062.txt +example_input_file8063.txt +example_input_file8064.txt +example_input_file8065.txt +example_input_file8066.txt +example_input_file8067.txt +example_input_file8068.txt +example_input_file8069.txt +example_input_file8070.txt +example_input_file8071.txt +example_input_file8072.txt +example_input_file8073.txt +example_input_file8074.txt +example_input_file8075.txt +example_input_file8076.txt +example_input_file8077.txt +example_input_file8078.txt +example_input_file8079.txt +example_input_file8080.txt +example_input_file8081.txt +example_input_file8082.txt +example_input_file8083.txt +example_input_file8084.txt +example_input_file8085.txt +example_input_file8086.txt +example_input_file8087.txt +example_input_file8088.txt +example_input_file8089.txt +example_input_file8090.txt +example_input_file8091.txt +example_input_file8092.txt +example_input_file8093.txt +example_input_file8094.txt +example_input_file8095.txt +example_input_file8096.txt +example_input_file8097.txt +example_input_file8098.txt +example_input_file8099.txt +example_input_file8100.txt +example_input_file8101.txt +example_input_file8102.txt +example_input_file8103.txt +example_input_file8104.txt +example_input_file8105.txt +example_input_file8106.txt +example_input_file8107.txt +example_input_file8108.txt +example_input_file8109.txt +example_input_file8110.txt +example_input_file8111.txt +example_input_file8112.txt +example_input_file8113.txt +example_input_file8114.txt +example_input_file8115.txt +example_input_file8116.txt +example_input_file8117.txt +example_input_file8118.txt +example_input_file8119.txt +example_input_file8120.txt +example_input_file8121.txt +example_input_file8122.txt +example_input_file8123.txt +example_input_file8124.txt +example_input_file8125.txt +example_input_file8126.txt +example_input_file8127.txt +example_input_file8128.txt +example_input_file8129.txt +example_input_file8130.txt +example_input_file8131.txt +example_input_file8132.txt +example_input_file8133.txt +example_input_file8134.txt +example_input_file8135.txt +example_input_file8136.txt +example_input_file8137.txt +example_input_file8138.txt +example_input_file8139.txt +example_input_file8140.txt +example_input_file8141.txt +example_input_file8142.txt +example_input_file8143.txt +example_input_file8144.txt +example_input_file8145.txt +example_input_file8146.txt +example_input_file8147.txt +example_input_file8148.txt +example_input_file8149.txt +example_input_file8150.txt +example_input_file8151.txt +example_input_file8152.txt +example_input_file8153.txt +example_input_file8154.txt +example_input_file8155.txt +example_input_file8156.txt +example_input_file8157.txt +example_input_file8158.txt +example_input_file8159.txt +example_input_file8160.txt +example_input_file8161.txt +example_input_file8162.txt +example_input_file8163.txt +example_input_file8164.txt +example_input_file8165.txt +example_input_file8166.txt +example_input_file8167.txt +example_input_file8168.txt +example_input_file8169.txt +example_input_file8170.txt +example_input_file8171.txt +example_input_file8172.txt +example_input_file8173.txt +example_input_file8174.txt +example_input_file8175.txt +example_input_file8176.txt +example_input_file8177.txt +example_input_file8178.txt +example_input_file8179.txt +example_input_file8180.txt +example_input_file8181.txt +example_input_file8182.txt +example_input_file8183.txt +example_input_file8184.txt +example_input_file8185.txt +example_input_file8186.txt +example_input_file8187.txt +example_input_file8188.txt +example_input_file8189.txt +example_input_file8190.txt +example_input_file8191.txt +example_input_file8192.txt +example_input_file8193.txt +example_input_file8194.txt +example_input_file8195.txt +example_input_file8196.txt +example_input_file8197.txt +example_input_file8198.txt +example_input_file8199.txt +example_input_file8200.txt +example_input_file8201.txt +example_input_file8202.txt +example_input_file8203.txt +example_input_file8204.txt +example_input_file8205.txt +example_input_file8206.txt +example_input_file8207.txt +example_input_file8208.txt +example_input_file8209.txt +example_input_file8210.txt +example_input_file8211.txt +example_input_file8212.txt +example_input_file8213.txt +example_input_file8214.txt +example_input_file8215.txt +example_input_file8216.txt +example_input_file8217.txt +example_input_file8218.txt +example_input_file8219.txt +example_input_file8220.txt +example_input_file8221.txt +example_input_file8222.txt +example_input_file8223.txt +example_input_file8224.txt +example_input_file8225.txt +example_input_file8226.txt +example_input_file8227.txt +example_input_file8228.txt +example_input_file8229.txt +example_input_file8230.txt +example_input_file8231.txt +example_input_file8232.txt +example_input_file8233.txt +example_input_file8234.txt +example_input_file8235.txt +example_input_file8236.txt +example_input_file8237.txt +example_input_file8238.txt +example_input_file8239.txt +example_input_file8240.txt +example_input_file8241.txt +example_input_file8242.txt +example_input_file8243.txt +example_input_file8244.txt +example_input_file8245.txt +example_input_file8246.txt +example_input_file8247.txt +example_input_file8248.txt +example_input_file8249.txt +example_input_file8250.txt +example_input_file8251.txt +example_input_file8252.txt +example_input_file8253.txt +example_input_file8254.txt +example_input_file8255.txt +example_input_file8256.txt +example_input_file8257.txt +example_input_file8258.txt +example_input_file8259.txt +example_input_file8260.txt +example_input_file8261.txt +example_input_file8262.txt +example_input_file8263.txt +example_input_file8264.txt +example_input_file8265.txt +example_input_file8266.txt +example_input_file8267.txt +example_input_file8268.txt +example_input_file8269.txt +example_input_file8270.txt +example_input_file8271.txt +example_input_file8272.txt +example_input_file8273.txt +example_input_file8274.txt +example_input_file8275.txt +example_input_file8276.txt +example_input_file8277.txt +example_input_file8278.txt +example_input_file8279.txt +example_input_file8280.txt +example_input_file8281.txt +example_input_file8282.txt +example_input_file8283.txt +example_input_file8284.txt +example_input_file8285.txt +example_input_file8286.txt +example_input_file8287.txt +example_input_file8288.txt +example_input_file8289.txt +example_input_file8290.txt +example_input_file8291.txt +example_input_file8292.txt +example_input_file8293.txt +example_input_file8294.txt +example_input_file8295.txt +example_input_file8296.txt +example_input_file8297.txt +example_input_file8298.txt +example_input_file8299.txt +example_input_file8300.txt +example_input_file8301.txt +example_input_file8302.txt +example_input_file8303.txt +example_input_file8304.txt +example_input_file8305.txt +example_input_file8306.txt +example_input_file8307.txt +example_input_file8308.txt +example_input_file8309.txt +example_input_file8310.txt +example_input_file8311.txt +example_input_file8312.txt +example_input_file8313.txt +example_input_file8314.txt +example_input_file8315.txt +example_input_file8316.txt +example_input_file8317.txt +example_input_file8318.txt +example_input_file8319.txt +example_input_file8320.txt +example_input_file8321.txt +example_input_file8322.txt +example_input_file8323.txt +example_input_file8324.txt +example_input_file8325.txt +example_input_file8326.txt +example_input_file8327.txt +example_input_file8328.txt +example_input_file8329.txt +example_input_file8330.txt +example_input_file8331.txt +example_input_file8332.txt +example_input_file8333.txt +example_input_file8334.txt +example_input_file8335.txt +example_input_file8336.txt +example_input_file8337.txt +example_input_file8338.txt +example_input_file8339.txt +example_input_file8340.txt +example_input_file8341.txt +example_input_file8342.txt +example_input_file8343.txt +example_input_file8344.txt +example_input_file8345.txt +example_input_file8346.txt +example_input_file8347.txt +example_input_file8348.txt +example_input_file8349.txt +example_input_file8350.txt +example_input_file8351.txt +example_input_file8352.txt +example_input_file8353.txt +example_input_file8354.txt +example_input_file8355.txt +example_input_file8356.txt +example_input_file8357.txt +example_input_file8358.txt +example_input_file8359.txt +example_input_file8360.txt +example_input_file8361.txt +example_input_file8362.txt +example_input_file8363.txt +example_input_file8364.txt +example_input_file8365.txt +example_input_file8366.txt +example_input_file8367.txt +example_input_file8368.txt +example_input_file8369.txt +example_input_file8370.txt +example_input_file8371.txt +example_input_file8372.txt +example_input_file8373.txt +example_input_file8374.txt +example_input_file8375.txt +example_input_file8376.txt +example_input_file8377.txt +example_input_file8378.txt +example_input_file8379.txt +example_input_file8380.txt +example_input_file8381.txt +example_input_file8382.txt +example_input_file8383.txt +example_input_file8384.txt +example_input_file8385.txt +example_input_file8386.txt +example_input_file8387.txt +example_input_file8388.txt +example_input_file8389.txt +example_input_file8390.txt +example_input_file8391.txt +example_input_file8392.txt +example_input_file8393.txt +example_input_file8394.txt +example_input_file8395.txt +example_input_file8396.txt +example_input_file8397.txt +example_input_file8398.txt +example_input_file8399.txt +example_input_file8400.txt +example_input_file8401.txt +example_input_file8402.txt +example_input_file8403.txt +example_input_file8404.txt +example_input_file8405.txt +example_input_file8406.txt +example_input_file8407.txt +example_input_file8408.txt +example_input_file8409.txt +example_input_file8410.txt +example_input_file8411.txt +example_input_file8412.txt +example_input_file8413.txt +example_input_file8414.txt +example_input_file8415.txt +example_input_file8416.txt +example_input_file8417.txt +example_input_file8418.txt +example_input_file8419.txt +example_input_file8420.txt +example_input_file8421.txt +example_input_file8422.txt +example_input_file8423.txt +example_input_file8424.txt +example_input_file8425.txt +example_input_file8426.txt +example_input_file8427.txt +example_input_file8428.txt +example_input_file8429.txt +example_input_file8430.txt +example_input_file8431.txt +example_input_file8432.txt +example_input_file8433.txt +example_input_file8434.txt +example_input_file8435.txt +example_input_file8436.txt +example_input_file8437.txt +example_input_file8438.txt +example_input_file8439.txt +example_input_file8440.txt +example_input_file8441.txt +example_input_file8442.txt +example_input_file8443.txt +example_input_file8444.txt +example_input_file8445.txt +example_input_file8446.txt +example_input_file8447.txt +example_input_file8448.txt +example_input_file8449.txt +example_input_file8450.txt +example_input_file8451.txt +example_input_file8452.txt +example_input_file8453.txt +example_input_file8454.txt +example_input_file8455.txt +example_input_file8456.txt +example_input_file8457.txt +example_input_file8458.txt +example_input_file8459.txt +example_input_file8460.txt +example_input_file8461.txt +example_input_file8462.txt +example_input_file8463.txt +example_input_file8464.txt +example_input_file8465.txt +example_input_file8466.txt +example_input_file8467.txt +example_input_file8468.txt +example_input_file8469.txt +example_input_file8470.txt +example_input_file8471.txt +example_input_file8472.txt +example_input_file8473.txt +example_input_file8474.txt +example_input_file8475.txt +example_input_file8476.txt +example_input_file8477.txt +example_input_file8478.txt +example_input_file8479.txt +example_input_file8480.txt +example_input_file8481.txt +example_input_file8482.txt +example_input_file8483.txt +example_input_file8484.txt +example_input_file8485.txt +example_input_file8486.txt +example_input_file8487.txt +example_input_file8488.txt +example_input_file8489.txt +example_input_file8490.txt +example_input_file8491.txt +example_input_file8492.txt +example_input_file8493.txt +example_input_file8494.txt +example_input_file8495.txt +example_input_file8496.txt +example_input_file8497.txt +example_input_file8498.txt +example_input_file8499.txt +example_input_file8500.txt +example_input_file8501.txt +example_input_file8502.txt +example_input_file8503.txt +example_input_file8504.txt +example_input_file8505.txt +example_input_file8506.txt +example_input_file8507.txt +example_input_file8508.txt +example_input_file8509.txt +example_input_file8510.txt +example_input_file8511.txt +example_input_file8512.txt +example_input_file8513.txt +example_input_file8514.txt +example_input_file8515.txt +example_input_file8516.txt +example_input_file8517.txt +example_input_file8518.txt +example_input_file8519.txt +example_input_file8520.txt +example_input_file8521.txt +example_input_file8522.txt +example_input_file8523.txt +example_input_file8524.txt +example_input_file8525.txt +example_input_file8526.txt +example_input_file8527.txt +example_input_file8528.txt +example_input_file8529.txt +example_input_file8530.txt +example_input_file8531.txt +example_input_file8532.txt +example_input_file8533.txt +example_input_file8534.txt +example_input_file8535.txt +example_input_file8536.txt +example_input_file8537.txt +example_input_file8538.txt +example_input_file8539.txt +example_input_file8540.txt +example_input_file8541.txt +example_input_file8542.txt +example_input_file8543.txt +example_input_file8544.txt +example_input_file8545.txt +example_input_file8546.txt +example_input_file8547.txt +example_input_file8548.txt +example_input_file8549.txt +example_input_file8550.txt +example_input_file8551.txt +example_input_file8552.txt +example_input_file8553.txt +example_input_file8554.txt +example_input_file8555.txt +example_input_file8556.txt +example_input_file8557.txt +example_input_file8558.txt +example_input_file8559.txt +example_input_file8560.txt +example_input_file8561.txt +example_input_file8562.txt +example_input_file8563.txt +example_input_file8564.txt +example_input_file8565.txt +example_input_file8566.txt +example_input_file8567.txt +example_input_file8568.txt +example_input_file8569.txt +example_input_file8570.txt +example_input_file8571.txt +example_input_file8572.txt +example_input_file8573.txt +example_input_file8574.txt +example_input_file8575.txt +example_input_file8576.txt +example_input_file8577.txt +example_input_file8578.txt +example_input_file8579.txt +example_input_file8580.txt +example_input_file8581.txt +example_input_file8582.txt +example_input_file8583.txt +example_input_file8584.txt +example_input_file8585.txt +example_input_file8586.txt +example_input_file8587.txt +example_input_file8588.txt +example_input_file8589.txt +example_input_file8590.txt +example_input_file8591.txt +example_input_file8592.txt +example_input_file8593.txt +example_input_file8594.txt +example_input_file8595.txt +example_input_file8596.txt +example_input_file8597.txt +example_input_file8598.txt +example_input_file8599.txt +example_input_file8600.txt +example_input_file8601.txt +example_input_file8602.txt +example_input_file8603.txt +example_input_file8604.txt +example_input_file8605.txt +example_input_file8606.txt +example_input_file8607.txt +example_input_file8608.txt +example_input_file8609.txt +example_input_file8610.txt +example_input_file8611.txt +example_input_file8612.txt +example_input_file8613.txt +example_input_file8614.txt +example_input_file8615.txt +example_input_file8616.txt +example_input_file8617.txt +example_input_file8618.txt +example_input_file8619.txt +example_input_file8620.txt +example_input_file8621.txt +example_input_file8622.txt +example_input_file8623.txt +example_input_file8624.txt +example_input_file8625.txt +example_input_file8626.txt +example_input_file8627.txt +example_input_file8628.txt +example_input_file8629.txt +example_input_file8630.txt +example_input_file8631.txt +example_input_file8632.txt +example_input_file8633.txt +example_input_file8634.txt +example_input_file8635.txt +example_input_file8636.txt +example_input_file8637.txt +example_input_file8638.txt +example_input_file8639.txt +example_input_file8640.txt +example_input_file8641.txt +example_input_file8642.txt +example_input_file8643.txt +example_input_file8644.txt +example_input_file8645.txt +example_input_file8646.txt +example_input_file8647.txt +example_input_file8648.txt +example_input_file8649.txt +example_input_file8650.txt +example_input_file8651.txt +example_input_file8652.txt +example_input_file8653.txt +example_input_file8654.txt +example_input_file8655.txt +example_input_file8656.txt +example_input_file8657.txt +example_input_file8658.txt +example_input_file8659.txt +example_input_file8660.txt +example_input_file8661.txt +example_input_file8662.txt +example_input_file8663.txt +example_input_file8664.txt +example_input_file8665.txt +example_input_file8666.txt +example_input_file8667.txt +example_input_file8668.txt +example_input_file8669.txt +example_input_file8670.txt +example_input_file8671.txt +example_input_file8672.txt +example_input_file8673.txt +example_input_file8674.txt +example_input_file8675.txt +example_input_file8676.txt +example_input_file8677.txt +example_input_file8678.txt +example_input_file8679.txt +example_input_file8680.txt +example_input_file8681.txt +example_input_file8682.txt +example_input_file8683.txt +example_input_file8684.txt +example_input_file8685.txt +example_input_file8686.txt +example_input_file8687.txt +example_input_file8688.txt +example_input_file8689.txt +example_input_file8690.txt +example_input_file8691.txt +example_input_file8692.txt +example_input_file8693.txt +example_input_file8694.txt +example_input_file8695.txt +example_input_file8696.txt +example_input_file8697.txt +example_input_file8698.txt +example_input_file8699.txt +example_input_file8700.txt +example_input_file8701.txt +example_input_file8702.txt +example_input_file8703.txt +example_input_file8704.txt +example_input_file8705.txt +example_input_file8706.txt +example_input_file8707.txt +example_input_file8708.txt +example_input_file8709.txt +example_input_file8710.txt +example_input_file8711.txt +example_input_file8712.txt +example_input_file8713.txt +example_input_file8714.txt +example_input_file8715.txt +example_input_file8716.txt +example_input_file8717.txt +example_input_file8718.txt +example_input_file8719.txt +example_input_file8720.txt +example_input_file8721.txt +example_input_file8722.txt +example_input_file8723.txt +example_input_file8724.txt +example_input_file8725.txt +example_input_file8726.txt +example_input_file8727.txt +example_input_file8728.txt +example_input_file8729.txt +example_input_file8730.txt +example_input_file8731.txt +example_input_file8732.txt +example_input_file8733.txt +example_input_file8734.txt +example_input_file8735.txt +example_input_file8736.txt +example_input_file8737.txt +example_input_file8738.txt +example_input_file8739.txt +example_input_file8740.txt +example_input_file8741.txt +example_input_file8742.txt +example_input_file8743.txt +example_input_file8744.txt +example_input_file8745.txt +example_input_file8746.txt +example_input_file8747.txt +example_input_file8748.txt +example_input_file8749.txt +example_input_file8750.txt +example_input_file8751.txt +example_input_file8752.txt +example_input_file8753.txt +example_input_file8754.txt +example_input_file8755.txt +example_input_file8756.txt +example_input_file8757.txt +example_input_file8758.txt +example_input_file8759.txt +example_input_file8760.txt +example_input_file8761.txt +example_input_file8762.txt +example_input_file8763.txt +example_input_file8764.txt +example_input_file8765.txt +example_input_file8766.txt +example_input_file8767.txt +example_input_file8768.txt +example_input_file8769.txt +example_input_file8770.txt +example_input_file8771.txt +example_input_file8772.txt +example_input_file8773.txt +example_input_file8774.txt +example_input_file8775.txt +example_input_file8776.txt +example_input_file8777.txt +example_input_file8778.txt +example_input_file8779.txt +example_input_file8780.txt +example_input_file8781.txt +example_input_file8782.txt +example_input_file8783.txt +example_input_file8784.txt +example_input_file8785.txt +example_input_file8786.txt +example_input_file8787.txt +example_input_file8788.txt +example_input_file8789.txt +example_input_file8790.txt +example_input_file8791.txt +example_input_file8792.txt +example_input_file8793.txt +example_input_file8794.txt +example_input_file8795.txt +example_input_file8796.txt +example_input_file8797.txt +example_input_file8798.txt +example_input_file8799.txt +example_input_file8800.txt +example_input_file8801.txt +example_input_file8802.txt +example_input_file8803.txt +example_input_file8804.txt +example_input_file8805.txt +example_input_file8806.txt +example_input_file8807.txt +example_input_file8808.txt +example_input_file8809.txt +example_input_file8810.txt +example_input_file8811.txt +example_input_file8812.txt +example_input_file8813.txt +example_input_file8814.txt +example_input_file8815.txt +example_input_file8816.txt +example_input_file8817.txt +example_input_file8818.txt +example_input_file8819.txt +example_input_file8820.txt +example_input_file8821.txt +example_input_file8822.txt +example_input_file8823.txt +example_input_file8824.txt +example_input_file8825.txt +example_input_file8826.txt +example_input_file8827.txt +example_input_file8828.txt +example_input_file8829.txt +example_input_file8830.txt +example_input_file8831.txt +example_input_file8832.txt +example_input_file8833.txt +example_input_file8834.txt +example_input_file8835.txt +example_input_file8836.txt +example_input_file8837.txt +example_input_file8838.txt +example_input_file8839.txt +example_input_file8840.txt +example_input_file8841.txt +example_input_file8842.txt +example_input_file8843.txt +example_input_file8844.txt +example_input_file8845.txt +example_input_file8846.txt +example_input_file8847.txt +example_input_file8848.txt +example_input_file8849.txt +example_input_file8850.txt +example_input_file8851.txt +example_input_file8852.txt +example_input_file8853.txt +example_input_file8854.txt +example_input_file8855.txt +example_input_file8856.txt +example_input_file8857.txt +example_input_file8858.txt +example_input_file8859.txt +example_input_file8860.txt +example_input_file8861.txt +example_input_file8862.txt +example_input_file8863.txt +example_input_file8864.txt +example_input_file8865.txt +example_input_file8866.txt +example_input_file8867.txt +example_input_file8868.txt +example_input_file8869.txt +example_input_file8870.txt +example_input_file8871.txt +example_input_file8872.txt +example_input_file8873.txt +example_input_file8874.txt +example_input_file8875.txt +example_input_file8876.txt +example_input_file8877.txt +example_input_file8878.txt +example_input_file8879.txt +example_input_file8880.txt +example_input_file8881.txt +example_input_file8882.txt +example_input_file8883.txt +example_input_file8884.txt +example_input_file8885.txt +example_input_file8886.txt +example_input_file8887.txt +example_input_file8888.txt +example_input_file8889.txt +example_input_file8890.txt +example_input_file8891.txt +example_input_file8892.txt +example_input_file8893.txt +example_input_file8894.txt +example_input_file8895.txt +example_input_file8896.txt +example_input_file8897.txt +example_input_file8898.txt +example_input_file8899.txt +example_input_file8900.txt +example_input_file8901.txt +example_input_file8902.txt +example_input_file8903.txt +example_input_file8904.txt +example_input_file8905.txt +example_input_file8906.txt +example_input_file8907.txt +example_input_file8908.txt +example_input_file8909.txt +example_input_file8910.txt +example_input_file8911.txt +example_input_file8912.txt +example_input_file8913.txt +example_input_file8914.txt +example_input_file8915.txt +example_input_file8916.txt +example_input_file8917.txt +example_input_file8918.txt +example_input_file8919.txt +example_input_file8920.txt +example_input_file8921.txt +example_input_file8922.txt +example_input_file8923.txt +example_input_file8924.txt +example_input_file8925.txt +example_input_file8926.txt +example_input_file8927.txt +example_input_file8928.txt +example_input_file8929.txt +example_input_file8930.txt +example_input_file8931.txt +example_input_file8932.txt +example_input_file8933.txt +example_input_file8934.txt +example_input_file8935.txt +example_input_file8936.txt +example_input_file8937.txt +example_input_file8938.txt +example_input_file8939.txt +example_input_file8940.txt +example_input_file8941.txt +example_input_file8942.txt +example_input_file8943.txt +example_input_file8944.txt +example_input_file8945.txt +example_input_file8946.txt +example_input_file8947.txt +example_input_file8948.txt +example_input_file8949.txt +example_input_file8950.txt +example_input_file8951.txt +example_input_file8952.txt +example_input_file8953.txt +example_input_file8954.txt +example_input_file8955.txt +example_input_file8956.txt +example_input_file8957.txt +example_input_file8958.txt +example_input_file8959.txt +example_input_file8960.txt +example_input_file8961.txt +example_input_file8962.txt +example_input_file8963.txt +example_input_file8964.txt +example_input_file8965.txt +example_input_file8966.txt +example_input_file8967.txt +example_input_file8968.txt +example_input_file8969.txt +example_input_file8970.txt +example_input_file8971.txt +example_input_file8972.txt +example_input_file8973.txt +example_input_file8974.txt +example_input_file8975.txt +example_input_file8976.txt +example_input_file8977.txt +example_input_file8978.txt +example_input_file8979.txt +example_input_file8980.txt +example_input_file8981.txt +example_input_file8982.txt +example_input_file8983.txt +example_input_file8984.txt +example_input_file8985.txt +example_input_file8986.txt +example_input_file8987.txt +example_input_file8988.txt +example_input_file8989.txt +example_input_file8990.txt +example_input_file8991.txt +example_input_file8992.txt +example_input_file8993.txt +example_input_file8994.txt +example_input_file8995.txt +example_input_file8996.txt +example_input_file8997.txt +example_input_file8998.txt +example_input_file8999.txt +example_input_file9000.txt +example_input_file9001.txt +example_input_file9002.txt +example_input_file9003.txt +example_input_file9004.txt +example_input_file9005.txt +example_input_file9006.txt +example_input_file9007.txt +example_input_file9008.txt +example_input_file9009.txt +example_input_file9010.txt +example_input_file9011.txt +example_input_file9012.txt +example_input_file9013.txt +example_input_file9014.txt +example_input_file9015.txt +example_input_file9016.txt +example_input_file9017.txt +example_input_file9018.txt +example_input_file9019.txt +example_input_file9020.txt +example_input_file9021.txt +example_input_file9022.txt +example_input_file9023.txt +example_input_file9024.txt +example_input_file9025.txt +example_input_file9026.txt +example_input_file9027.txt +example_input_file9028.txt +example_input_file9029.txt +example_input_file9030.txt +example_input_file9031.txt +example_input_file9032.txt +example_input_file9033.txt +example_input_file9034.txt +example_input_file9035.txt +example_input_file9036.txt +example_input_file9037.txt +example_input_file9038.txt +example_input_file9039.txt +example_input_file9040.txt +example_input_file9041.txt +example_input_file9042.txt +example_input_file9043.txt +example_input_file9044.txt +example_input_file9045.txt +example_input_file9046.txt +example_input_file9047.txt +example_input_file9048.txt +example_input_file9049.txt +example_input_file9050.txt +example_input_file9051.txt +example_input_file9052.txt +example_input_file9053.txt +example_input_file9054.txt +example_input_file9055.txt +example_input_file9056.txt +example_input_file9057.txt +example_input_file9058.txt +example_input_file9059.txt +example_input_file9060.txt +example_input_file9061.txt +example_input_file9062.txt +example_input_file9063.txt +example_input_file9064.txt +example_input_file9065.txt +example_input_file9066.txt +example_input_file9067.txt +example_input_file9068.txt +example_input_file9069.txt +example_input_file9070.txt +example_input_file9071.txt +example_input_file9072.txt +example_input_file9073.txt +example_input_file9074.txt +example_input_file9075.txt +example_input_file9076.txt +example_input_file9077.txt +example_input_file9078.txt +example_input_file9079.txt +example_input_file9080.txt +example_input_file9081.txt +example_input_file9082.txt +example_input_file9083.txt +example_input_file9084.txt +example_input_file9085.txt +example_input_file9086.txt +example_input_file9087.txt +example_input_file9088.txt +example_input_file9089.txt +example_input_file9090.txt +example_input_file9091.txt +example_input_file9092.txt +example_input_file9093.txt +example_input_file9094.txt +example_input_file9095.txt +example_input_file9096.txt +example_input_file9097.txt +example_input_file9098.txt +example_input_file9099.txt +example_input_file9100.txt +example_input_file9101.txt +example_input_file9102.txt +example_input_file9103.txt +example_input_file9104.txt +example_input_file9105.txt +example_input_file9106.txt +example_input_file9107.txt +example_input_file9108.txt +example_input_file9109.txt +example_input_file9110.txt +example_input_file9111.txt +example_input_file9112.txt +example_input_file9113.txt +example_input_file9114.txt +example_input_file9115.txt +example_input_file9116.txt +example_input_file9117.txt +example_input_file9118.txt +example_input_file9119.txt +example_input_file9120.txt +example_input_file9121.txt +example_input_file9122.txt +example_input_file9123.txt +example_input_file9124.txt +example_input_file9125.txt +example_input_file9126.txt +example_input_file9127.txt +example_input_file9128.txt +example_input_file9129.txt +example_input_file9130.txt +example_input_file9131.txt +example_input_file9132.txt +example_input_file9133.txt +example_input_file9134.txt +example_input_file9135.txt +example_input_file9136.txt +example_input_file9137.txt +example_input_file9138.txt +example_input_file9139.txt +example_input_file9140.txt +example_input_file9141.txt +example_input_file9142.txt +example_input_file9143.txt +example_input_file9144.txt +example_input_file9145.txt +example_input_file9146.txt +example_input_file9147.txt +example_input_file9148.txt +example_input_file9149.txt +example_input_file9150.txt +example_input_file9151.txt +example_input_file9152.txt +example_input_file9153.txt +example_input_file9154.txt +example_input_file9155.txt +example_input_file9156.txt +example_input_file9157.txt +example_input_file9158.txt +example_input_file9159.txt +example_input_file9160.txt +example_input_file9161.txt +example_input_file9162.txt +example_input_file9163.txt +example_input_file9164.txt +example_input_file9165.txt +example_input_file9166.txt +example_input_file9167.txt +example_input_file9168.txt +example_input_file9169.txt +example_input_file9170.txt +example_input_file9171.txt +example_input_file9172.txt +example_input_file9173.txt +example_input_file9174.txt +example_input_file9175.txt +example_input_file9176.txt +example_input_file9177.txt +example_input_file9178.txt +example_input_file9179.txt +example_input_file9180.txt +example_input_file9181.txt +example_input_file9182.txt +example_input_file9183.txt +example_input_file9184.txt +example_input_file9185.txt +example_input_file9186.txt +example_input_file9187.txt +example_input_file9188.txt +example_input_file9189.txt +example_input_file9190.txt +example_input_file9191.txt +example_input_file9192.txt +example_input_file9193.txt +example_input_file9194.txt +example_input_file9195.txt +example_input_file9196.txt +example_input_file9197.txt +example_input_file9198.txt +example_input_file9199.txt +example_input_file9200.txt +example_input_file9201.txt +example_input_file9202.txt +example_input_file9203.txt +example_input_file9204.txt +example_input_file9205.txt +example_input_file9206.txt +example_input_file9207.txt +example_input_file9208.txt +example_input_file9209.txt +example_input_file9210.txt +example_input_file9211.txt +example_input_file9212.txt +example_input_file9213.txt +example_input_file9214.txt +example_input_file9215.txt +example_input_file9216.txt +example_input_file9217.txt +example_input_file9218.txt +example_input_file9219.txt +example_input_file9220.txt +example_input_file9221.txt +example_input_file9222.txt +example_input_file9223.txt +example_input_file9224.txt +example_input_file9225.txt +example_input_file9226.txt +example_input_file9227.txt +example_input_file9228.txt +example_input_file9229.txt +example_input_file9230.txt +example_input_file9231.txt +example_input_file9232.txt +example_input_file9233.txt +example_input_file9234.txt +example_input_file9235.txt +example_input_file9236.txt +example_input_file9237.txt +example_input_file9238.txt +example_input_file9239.txt +example_input_file9240.txt +example_input_file9241.txt +example_input_file9242.txt +example_input_file9243.txt +example_input_file9244.txt +example_input_file9245.txt +example_input_file9246.txt +example_input_file9247.txt +example_input_file9248.txt +example_input_file9249.txt +example_input_file9250.txt +example_input_file9251.txt +example_input_file9252.txt +example_input_file9253.txt +example_input_file9254.txt +example_input_file9255.txt +example_input_file9256.txt +example_input_file9257.txt +example_input_file9258.txt +example_input_file9259.txt +example_input_file9260.txt +example_input_file9261.txt +example_input_file9262.txt +example_input_file9263.txt +example_input_file9264.txt +example_input_file9265.txt +example_input_file9266.txt +example_input_file9267.txt +example_input_file9268.txt +example_input_file9269.txt +example_input_file9270.txt +example_input_file9271.txt +example_input_file9272.txt +example_input_file9273.txt +example_input_file9274.txt +example_input_file9275.txt +example_input_file9276.txt +example_input_file9277.txt +example_input_file9278.txt +example_input_file9279.txt +example_input_file9280.txt +example_input_file9281.txt +example_input_file9282.txt +example_input_file9283.txt +example_input_file9284.txt +example_input_file9285.txt +example_input_file9286.txt +example_input_file9287.txt +example_input_file9288.txt +example_input_file9289.txt +example_input_file9290.txt +example_input_file9291.txt +example_input_file9292.txt +example_input_file9293.txt +example_input_file9294.txt +example_input_file9295.txt +example_input_file9296.txt +example_input_file9297.txt +example_input_file9298.txt +example_input_file9299.txt +example_input_file9300.txt +example_input_file9301.txt +example_input_file9302.txt +example_input_file9303.txt +example_input_file9304.txt +example_input_file9305.txt +example_input_file9306.txt +example_input_file9307.txt +example_input_file9308.txt +example_input_file9309.txt +example_input_file9310.txt +example_input_file9311.txt +example_input_file9312.txt +example_input_file9313.txt +example_input_file9314.txt +example_input_file9315.txt +example_input_file9316.txt +example_input_file9317.txt +example_input_file9318.txt +example_input_file9319.txt +example_input_file9320.txt +example_input_file9321.txt +example_input_file9322.txt +example_input_file9323.txt +example_input_file9324.txt +example_input_file9325.txt +example_input_file9326.txt +example_input_file9327.txt +example_input_file9328.txt +example_input_file9329.txt +example_input_file9330.txt +example_input_file9331.txt +example_input_file9332.txt +example_input_file9333.txt +example_input_file9334.txt +example_input_file9335.txt +example_input_file9336.txt +example_input_file9337.txt +example_input_file9338.txt +example_input_file9339.txt +example_input_file9340.txt +example_input_file9341.txt +example_input_file9342.txt +example_input_file9343.txt +example_input_file9344.txt +example_input_file9345.txt +example_input_file9346.txt +example_input_file9347.txt +example_input_file9348.txt +example_input_file9349.txt +example_input_file9350.txt +example_input_file9351.txt +example_input_file9352.txt +example_input_file9353.txt +example_input_file9354.txt +example_input_file9355.txt +example_input_file9356.txt +example_input_file9357.txt +example_input_file9358.txt +example_input_file9359.txt +example_input_file9360.txt +example_input_file9361.txt +example_input_file9362.txt +example_input_file9363.txt +example_input_file9364.txt +example_input_file9365.txt +example_input_file9366.txt +example_input_file9367.txt +example_input_file9368.txt +example_input_file9369.txt +example_input_file9370.txt +example_input_file9371.txt +example_input_file9372.txt +example_input_file9373.txt +example_input_file9374.txt +example_input_file9375.txt +example_input_file9376.txt +example_input_file9377.txt +example_input_file9378.txt +example_input_file9379.txt +example_input_file9380.txt +example_input_file9381.txt +example_input_file9382.txt +example_input_file9383.txt +example_input_file9384.txt +example_input_file9385.txt +example_input_file9386.txt +example_input_file9387.txt +example_input_file9388.txt +example_input_file9389.txt +example_input_file9390.txt +example_input_file9391.txt +example_input_file9392.txt +example_input_file9393.txt +example_input_file9394.txt +example_input_file9395.txt +example_input_file9396.txt +example_input_file9397.txt +example_input_file9398.txt +example_input_file9399.txt +example_input_file9400.txt +example_input_file9401.txt +example_input_file9402.txt +example_input_file9403.txt +example_input_file9404.txt +example_input_file9405.txt +example_input_file9406.txt +example_input_file9407.txt +example_input_file9408.txt +example_input_file9409.txt +example_input_file9410.txt +example_input_file9411.txt +example_input_file9412.txt +example_input_file9413.txt +example_input_file9414.txt +example_input_file9415.txt +example_input_file9416.txt +example_input_file9417.txt +example_input_file9418.txt +example_input_file9419.txt +example_input_file9420.txt +example_input_file9421.txt +example_input_file9422.txt +example_input_file9423.txt +example_input_file9424.txt +example_input_file9425.txt +example_input_file9426.txt +example_input_file9427.txt +example_input_file9428.txt +example_input_file9429.txt +example_input_file9430.txt +example_input_file9431.txt +example_input_file9432.txt +example_input_file9433.txt +example_input_file9434.txt +example_input_file9435.txt +example_input_file9436.txt +example_input_file9437.txt +example_input_file9438.txt +example_input_file9439.txt +example_input_file9440.txt +example_input_file9441.txt +example_input_file9442.txt +example_input_file9443.txt +example_input_file9444.txt +example_input_file9445.txt +example_input_file9446.txt +example_input_file9447.txt +example_input_file9448.txt +example_input_file9449.txt +example_input_file9450.txt +example_input_file9451.txt +example_input_file9452.txt +example_input_file9453.txt +example_input_file9454.txt +example_input_file9455.txt +example_input_file9456.txt +example_input_file9457.txt +example_input_file9458.txt +example_input_file9459.txt +example_input_file9460.txt +example_input_file9461.txt +example_input_file9462.txt +example_input_file9463.txt +example_input_file9464.txt +example_input_file9465.txt +example_input_file9466.txt +example_input_file9467.txt +example_input_file9468.txt +example_input_file9469.txt +example_input_file9470.txt +example_input_file9471.txt +example_input_file9472.txt +example_input_file9473.txt +example_input_file9474.txt +example_input_file9475.txt +example_input_file9476.txt +example_input_file9477.txt +example_input_file9478.txt +example_input_file9479.txt +example_input_file9480.txt +example_input_file9481.txt +example_input_file9482.txt +example_input_file9483.txt +example_input_file9484.txt +example_input_file9485.txt +example_input_file9486.txt +example_input_file9487.txt +example_input_file9488.txt +example_input_file9489.txt +example_input_file9490.txt +example_input_file9491.txt +example_input_file9492.txt +example_input_file9493.txt +example_input_file9494.txt +example_input_file9495.txt +example_input_file9496.txt +example_input_file9497.txt +example_input_file9498.txt +example_input_file9499.txt +example_input_file9500.txt +example_input_file9501.txt +example_input_file9502.txt +example_input_file9503.txt +example_input_file9504.txt +example_input_file9505.txt +example_input_file9506.txt +example_input_file9507.txt +example_input_file9508.txt +example_input_file9509.txt +example_input_file9510.txt +example_input_file9511.txt +example_input_file9512.txt +example_input_file9513.txt +example_input_file9514.txt +example_input_file9515.txt +example_input_file9516.txt +example_input_file9517.txt +example_input_file9518.txt +example_input_file9519.txt +example_input_file9520.txt +example_input_file9521.txt +example_input_file9522.txt +example_input_file9523.txt +example_input_file9524.txt +example_input_file9525.txt +example_input_file9526.txt +example_input_file9527.txt +example_input_file9528.txt +example_input_file9529.txt +example_input_file9530.txt +example_input_file9531.txt +example_input_file9532.txt +example_input_file9533.txt +example_input_file9534.txt +example_input_file9535.txt +example_input_file9536.txt +example_input_file9537.txt +example_input_file9538.txt +example_input_file9539.txt +example_input_file9540.txt +example_input_file9541.txt +example_input_file9542.txt +example_input_file9543.txt +example_input_file9544.txt +example_input_file9545.txt +example_input_file9546.txt +example_input_file9547.txt +example_input_file9548.txt +example_input_file9549.txt +example_input_file9550.txt +example_input_file9551.txt +example_input_file9552.txt +example_input_file9553.txt +example_input_file9554.txt +example_input_file9555.txt +example_input_file9556.txt +example_input_file9557.txt +example_input_file9558.txt +example_input_file9559.txt +example_input_file9560.txt +example_input_file9561.txt +example_input_file9562.txt +example_input_file9563.txt +example_input_file9564.txt +example_input_file9565.txt +example_input_file9566.txt +example_input_file9567.txt +example_input_file9568.txt +example_input_file9569.txt +example_input_file9570.txt +example_input_file9571.txt +example_input_file9572.txt +example_input_file9573.txt +example_input_file9574.txt +example_input_file9575.txt +example_input_file9576.txt +example_input_file9577.txt +example_input_file9578.txt +example_input_file9579.txt +example_input_file9580.txt +example_input_file9581.txt +example_input_file9582.txt +example_input_file9583.txt +example_input_file9584.txt +example_input_file9585.txt +example_input_file9586.txt +example_input_file9587.txt +example_input_file9588.txt +example_input_file9589.txt +example_input_file9590.txt +example_input_file9591.txt +example_input_file9592.txt +example_input_file9593.txt +example_input_file9594.txt +example_input_file9595.txt +example_input_file9596.txt +example_input_file9597.txt +example_input_file9598.txt +example_input_file9599.txt +example_input_file9600.txt +example_input_file9601.txt +example_input_file9602.txt +example_input_file9603.txt +example_input_file9604.txt +example_input_file9605.txt +example_input_file9606.txt +example_input_file9607.txt +example_input_file9608.txt +example_input_file9609.txt +example_input_file9610.txt +example_input_file9611.txt +example_input_file9612.txt +example_input_file9613.txt +example_input_file9614.txt +example_input_file9615.txt +example_input_file9616.txt +example_input_file9617.txt +example_input_file9618.txt +example_input_file9619.txt +example_input_file9620.txt +example_input_file9621.txt +example_input_file9622.txt +example_input_file9623.txt +example_input_file9624.txt +example_input_file9625.txt +example_input_file9626.txt +example_input_file9627.txt +example_input_file9628.txt +example_input_file9629.txt +example_input_file9630.txt +example_input_file9631.txt +example_input_file9632.txt +example_input_file9633.txt +example_input_file9634.txt +example_input_file9635.txt +example_input_file9636.txt +example_input_file9637.txt +example_input_file9638.txt +example_input_file9639.txt +example_input_file9640.txt +example_input_file9641.txt +example_input_file9642.txt +example_input_file9643.txt +example_input_file9644.txt +example_input_file9645.txt +example_input_file9646.txt +example_input_file9647.txt +example_input_file9648.txt +example_input_file9649.txt +example_input_file9650.txt +example_input_file9651.txt +example_input_file9652.txt +example_input_file9653.txt +example_input_file9654.txt +example_input_file9655.txt +example_input_file9656.txt +example_input_file9657.txt +example_input_file9658.txt +example_input_file9659.txt +example_input_file9660.txt +example_input_file9661.txt +example_input_file9662.txt +example_input_file9663.txt +example_input_file9664.txt +example_input_file9665.txt +example_input_file9666.txt +example_input_file9667.txt +example_input_file9668.txt +example_input_file9669.txt +example_input_file9670.txt +example_input_file9671.txt +example_input_file9672.txt +example_input_file9673.txt +example_input_file9674.txt +example_input_file9675.txt +example_input_file9676.txt +example_input_file9677.txt +example_input_file9678.txt +example_input_file9679.txt +example_input_file9680.txt +example_input_file9681.txt +example_input_file9682.txt +example_input_file9683.txt +example_input_file9684.txt +example_input_file9685.txt +example_input_file9686.txt +example_input_file9687.txt +example_input_file9688.txt +example_input_file9689.txt +example_input_file9690.txt +example_input_file9691.txt +example_input_file9692.txt +example_input_file9693.txt +example_input_file9694.txt +example_input_file9695.txt +example_input_file9696.txt +example_input_file9697.txt +example_input_file9698.txt +example_input_file9699.txt +example_input_file9700.txt +example_input_file9701.txt +example_input_file9702.txt +example_input_file9703.txt +example_input_file9704.txt +example_input_file9705.txt +example_input_file9706.txt +example_input_file9707.txt +example_input_file9708.txt +example_input_file9709.txt +example_input_file9710.txt +example_input_file9711.txt +example_input_file9712.txt +example_input_file9713.txt +example_input_file9714.txt +example_input_file9715.txt +example_input_file9716.txt +example_input_file9717.txt +example_input_file9718.txt +example_input_file9719.txt +example_input_file9720.txt +example_input_file9721.txt +example_input_file9722.txt +example_input_file9723.txt +example_input_file9724.txt +example_input_file9725.txt +example_input_file9726.txt +example_input_file9727.txt +example_input_file9728.txt +example_input_file9729.txt +example_input_file9730.txt +example_input_file9731.txt +example_input_file9732.txt +example_input_file9733.txt +example_input_file9734.txt +example_input_file9735.txt +example_input_file9736.txt +example_input_file9737.txt +example_input_file9738.txt +example_input_file9739.txt +example_input_file9740.txt +example_input_file9741.txt +example_input_file9742.txt +example_input_file9743.txt +example_input_file9744.txt +example_input_file9745.txt +example_input_file9746.txt +example_input_file9747.txt +example_input_file9748.txt +example_input_file9749.txt +example_input_file9750.txt +example_input_file9751.txt +example_input_file9752.txt +example_input_file9753.txt +example_input_file9754.txt +example_input_file9755.txt +example_input_file9756.txt +example_input_file9757.txt +example_input_file9758.txt +example_input_file9759.txt +example_input_file9760.txt +example_input_file9761.txt +example_input_file9762.txt +example_input_file9763.txt +example_input_file9764.txt +example_input_file9765.txt +example_input_file9766.txt +example_input_file9767.txt +example_input_file9768.txt +example_input_file9769.txt +example_input_file9770.txt +example_input_file9771.txt +example_input_file9772.txt +example_input_file9773.txt +example_input_file9774.txt +example_input_file9775.txt +example_input_file9776.txt +example_input_file9777.txt +example_input_file9778.txt +example_input_file9779.txt +example_input_file9780.txt +example_input_file9781.txt +example_input_file9782.txt +example_input_file9783.txt +example_input_file9784.txt +example_input_file9785.txt +example_input_file9786.txt +example_input_file9787.txt +example_input_file9788.txt +example_input_file9789.txt +example_input_file9790.txt +example_input_file9791.txt +example_input_file9792.txt +example_input_file9793.txt +example_input_file9794.txt +example_input_file9795.txt +example_input_file9796.txt +example_input_file9797.txt +example_input_file9798.txt +example_input_file9799.txt +example_input_file9800.txt +example_input_file9801.txt +example_input_file9802.txt +example_input_file9803.txt +example_input_file9804.txt +example_input_file9805.txt +example_input_file9806.txt +example_input_file9807.txt +example_input_file9808.txt +example_input_file9809.txt +example_input_file9810.txt +example_input_file9811.txt +example_input_file9812.txt +example_input_file9813.txt +example_input_file9814.txt +example_input_file9815.txt +example_input_file9816.txt +example_input_file9817.txt +example_input_file9818.txt +example_input_file9819.txt +example_input_file9820.txt +example_input_file9821.txt +example_input_file9822.txt +example_input_file9823.txt +example_input_file9824.txt +example_input_file9825.txt +example_input_file9826.txt +example_input_file9827.txt +example_input_file9828.txt +example_input_file9829.txt +example_input_file9830.txt +example_input_file9831.txt +example_input_file9832.txt +example_input_file9833.txt +example_input_file9834.txt +example_input_file9835.txt +example_input_file9836.txt +example_input_file9837.txt +example_input_file9838.txt +example_input_file9839.txt +example_input_file9840.txt +example_input_file9841.txt +example_input_file9842.txt +example_input_file9843.txt +example_input_file9844.txt +example_input_file9845.txt +example_input_file9846.txt +example_input_file9847.txt +example_input_file9848.txt +example_input_file9849.txt +example_input_file9850.txt +example_input_file9851.txt +example_input_file9852.txt +example_input_file9853.txt +example_input_file9854.txt +example_input_file9855.txt +example_input_file9856.txt +example_input_file9857.txt +example_input_file9858.txt +example_input_file9859.txt +example_input_file9860.txt +example_input_file9861.txt +example_input_file9862.txt +example_input_file9863.txt +example_input_file9864.txt +example_input_file9865.txt +example_input_file9866.txt +example_input_file9867.txt +example_input_file9868.txt +example_input_file9869.txt +example_input_file9870.txt +example_input_file9871.txt +example_input_file9872.txt +example_input_file9873.txt +example_input_file9874.txt +example_input_file9875.txt +example_input_file9876.txt +example_input_file9877.txt +example_input_file9878.txt +example_input_file9879.txt +example_input_file9880.txt +example_input_file9881.txt +example_input_file9882.txt +example_input_file9883.txt +example_input_file9884.txt +example_input_file9885.txt +example_input_file9886.txt +example_input_file9887.txt +example_input_file9888.txt +example_input_file9889.txt +example_input_file9890.txt +example_input_file9891.txt +example_input_file9892.txt +example_input_file9893.txt +example_input_file9894.txt +example_input_file9895.txt +example_input_file9896.txt +example_input_file9897.txt +example_input_file9898.txt +example_input_file9899.txt +example_input_file9900.txt +example_input_file9901.txt +example_input_file9902.txt +example_input_file9903.txt +example_input_file9904.txt +example_input_file9905.txt +example_input_file9906.txt +example_input_file9907.txt +example_input_file9908.txt +example_input_file9909.txt +example_input_file9910.txt +example_input_file9911.txt +example_input_file9912.txt +example_input_file9913.txt +example_input_file9914.txt +example_input_file9915.txt +example_input_file9916.txt +example_input_file9917.txt +example_input_file9918.txt +example_input_file9919.txt +example_input_file9920.txt +example_input_file9921.txt +example_input_file9922.txt +example_input_file9923.txt +example_input_file9924.txt +example_input_file9925.txt +example_input_file9926.txt +example_input_file9927.txt +example_input_file9928.txt +example_input_file9929.txt +example_input_file9930.txt +example_input_file9931.txt +example_input_file9932.txt +example_input_file9933.txt +example_input_file9934.txt +example_input_file9935.txt +example_input_file9936.txt +example_input_file9937.txt +example_input_file9938.txt +example_input_file9939.txt +example_input_file9940.txt +example_input_file9941.txt +example_input_file9942.txt +example_input_file9943.txt +example_input_file9944.txt +example_input_file9945.txt +example_input_file9946.txt +example_input_file9947.txt +example_input_file9948.txt +example_input_file9949.txt +example_input_file9950.txt +example_input_file9951.txt +example_input_file9952.txt +example_input_file9953.txt +example_input_file9954.txt +example_input_file9955.txt +example_input_file9956.txt +example_input_file9957.txt +example_input_file9958.txt +example_input_file9959.txt +example_input_file9960.txt +example_input_file9961.txt +example_input_file9962.txt +example_input_file9963.txt +example_input_file9964.txt +example_input_file9965.txt +example_input_file9966.txt +example_input_file9967.txt +example_input_file9968.txt +example_input_file9969.txt +example_input_file9970.txt +example_input_file9971.txt +example_input_file9972.txt +example_input_file9973.txt +example_input_file9974.txt +example_input_file9975.txt +example_input_file9976.txt +example_input_file9977.txt +example_input_file9978.txt +example_input_file9979.txt +example_input_file9980.txt +example_input_file9981.txt +example_input_file9982.txt +example_input_file9983.txt +example_input_file9984.txt +example_input_file9985.txt +example_input_file9986.txt +example_input_file9987.txt +example_input_file9988.txt +example_input_file9989.txt +example_input_file9990.txt +example_input_file9991.txt +example_input_file9992.txt +example_input_file9993.txt +example_input_file9994.txt +example_input_file9995.txt +example_input_file9996.txt +example_input_file9997.txt +example_input_file9998.txt +example_input_file9999.txt diff --git a/tests/wf/input_named_id.cwl b/tests/wf/input_named_id.cwl old mode 100644 new mode 100755 index e559f967b..82a0353aa --- a/tests/wf/input_named_id.cwl +++ b/tests/wf/input_named_id.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner label: FeatureFinderIdentification doc: "" inputs: diff --git a/tests/wf/iwd-container-entryname1.cwl b/tests/wf/iwd-container-entryname1.cwl new file mode 100644 index 000000000..3fe16c9e8 --- /dev/null +++ b/tests/wf/iwd-container-entryname1.cwl @@ -0,0 +1,23 @@ +cwlVersion: v1.2 +class: CommandLineTool +doc: | + When executing in a container, entryname can have an absolute path + to a mount location inside the container. +inputs: + filelist: File +outputs: + head: + type: File + outputBinding: + glob: head.txt +requirements: + DockerRequirement: + dockerPull: docker.io/debian:stable-slim + dockerOutputDirectory: /output + InitialWorkDirRequirement: + listing: + - entryname: /tmp2j3y7rpb/input/stuff.txt # Give it a weird prefix to minimize chance of conflict with a real file + entry: $(inputs.filelist) + ShellCommandRequirement: {} +arguments: + - {shellQuote: false, valueFrom: "head -n10 /tmp2j3y7rpb/input/stuff.txt > /output/head.txt"} diff --git a/tests/wf/iwd-container-entryname3.cwl b/tests/wf/iwd-container-entryname3.cwl new file mode 100644 index 000000000..cfa14e631 --- /dev/null +++ b/tests/wf/iwd-container-entryname3.cwl @@ -0,0 +1,24 @@ +cwlVersion: v1.2 +class: CommandLineTool +doc: | + Must fail if entryname is an absolute path and DockerRequirement is + not in the 'requirements' section. +inputs: + filelist: File +outputs: + head: + type: File + outputBinding: + glob: head.txt +hints: + DockerRequirement: + dockerPull: docker.io/debian:stable-slim + dockerOutputDirectory: /output +requirements: + InitialWorkDirRequirement: + listing: + - entryname: /tmp2j3y7rpb/input/stuff.txt # Give it a weird prefix to minimize chance of conflict with a real file + entry: $(inputs.filelist) + ShellCommandRequirement: {} +arguments: + - {shellQuote: false, valueFrom: "head -n10 /tmp2j3y7rpb/input/stuff.txt > /output/head.txt"} diff --git a/tests/wf/iwd-passthrough1.cwl b/tests/wf/iwd-passthrough1.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/iwdr-empty.cwl b/tests/wf/iwdr-empty.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/iwdr-entry.cwl b/tests/wf/iwdr-entry.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/iwdr-passthrough-successive.cwl b/tests/wf/iwdr-passthrough-successive.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/literalfile.cwl b/tests/wf/literalfile.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/loadContents-input.yml b/tests/wf/loadContents-input.yml new file mode 100644 index 000000000..a44fd27ca --- /dev/null +++ b/tests/wf/loadContents-input.yml @@ -0,0 +1,3 @@ +filelist: + class: File + location: inp-filelist.txt diff --git a/tests/wf/malformed_outputs.cwl b/tests/wf/malformed_outputs.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/missing-tool.cwl b/tests/wf/missing-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_env.cwl b/tests/wf/mpi_env.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_expr.cwl b/tests/wf/mpi_expr.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_line_count.cwl b/tests/wf/mpi_line_count.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_simple.cwl b/tests/wf/mpi_simple.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/mpi_simple_wf.cwl b/tests/wf/mpi_simple_wf.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/nested.cwl b/tests/wf/nested.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/networkaccess-fail.cwl b/tests/wf/networkaccess-fail.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/networkaccess.cwl b/tests/wf/networkaccess.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/no-parameters-echo.cwl b/tests/wf/no-parameters-echo.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/nvidia-smi-cc.cwl b/tests/wf/nvidia-smi-cc.cwl old mode 100644 new mode 100755 index a4f315b0e..92e781d26 --- a/tests/wf/nvidia-smi-cc.cwl +++ b/tests/wf/nvidia-smi-cc.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/nvidia-smi-container.cwl b/tests/wf/nvidia-smi-container.cwl old mode 100644 new mode 100755 index 84fd72d83..22092fae4 --- a/tests/wf/nvidia-smi-container.cwl +++ b/tests/wf/nvidia-smi-container.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/nvidia-smi-max.cwl b/tests/wf/nvidia-smi-max.cwl old mode 100644 new mode 100755 index d3d4d5e9c..1f8687d4d --- a/tests/wf/nvidia-smi-max.cwl +++ b/tests/wf/nvidia-smi-max.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/nvidia-smi-range.cwl b/tests/wf/nvidia-smi-range.cwl old mode 100644 new mode 100755 index 19d3ea43c..2e131a373 --- a/tests/wf/nvidia-smi-range.cwl +++ b/tests/wf/nvidia-smi-range.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/nvidia-smi.cwl b/tests/wf/nvidia-smi.cwl old mode 100644 new mode 100755 index 8e227d0c5..8a17f8e17 --- a/tests/wf/nvidia-smi.cwl +++ b/tests/wf/nvidia-smi.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool $namespaces: diff --git a/tests/wf/operation/abstract-cosifer.cwl b/tests/wf/operation/abstract-cosifer.cwl old mode 100644 new mode 100755 index 057620844..2a50cded7 --- a/tests/wf/operation/abstract-cosifer.cwl +++ b/tests/wf/operation/abstract-cosifer.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: Operation cwlVersion: v1.2 diff --git a/tests/wf/operation/operation-single.cwl b/tests/wf/operation/operation-single.cwl old mode 100644 new mode 100755 index 2119bf02e..63b6a6bf0 --- a/tests/wf/operation/operation-single.cwl +++ b/tests/wf/operation/operation-single.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: Workflow cwlVersion: v1.2 id: abstract_cosifer_workflow diff --git a/tests/wf/optional-numerical-output-0.cwl b/tests/wf/optional-numerical-output-0.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/optional_src_mandatory_sink.cwl b/tests/wf/optional_src_mandatory_sink.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/packed-with-loadlisting.cwl b/tests/wf/packed-with-loadlisting.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/packed_no_main.cwl b/tests/wf/packed_no_main.cwl old mode 100644 new mode 100755 index 8307a083e..aafd0de58 --- a/tests/wf/packed_no_main.cwl +++ b/tests/wf/packed_no_main.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 $graph: - id: echo diff --git a/tests/wf/paramref_arguments_roundtrip.cwl b/tests/wf/paramref_arguments_roundtrip.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/paramref_arguments_self.cwl b/tests/wf/paramref_arguments_self.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/record_outputeval.cwl b/tests/wf/record_outputeval.cwl old mode 100644 new mode 100755 index 45daf9b4d..e7b59cb6c --- a/tests/wf/record_outputeval.cwl +++ b/tests/wf/record_outputeval.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool requirements: diff --git a/tests/wf/resreq_expr_float_v1_0.cwl b/tests/wf/resreq_expr_float_v1_0.cwl old mode 100644 new mode 100755 index d5ea08cde..a36877c0f --- a/tests/wf/resreq_expr_float_v1_0.cwl +++ b/tests/wf/resreq_expr_float_v1_0.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: CommandLineTool requirements: diff --git a/tests/wf/resreq_expr_float_v1_2.cwl b/tests/wf/resreq_expr_float_v1_2.cwl old mode 100644 new mode 100755 index cfb1b2a13..ca72364b7 --- a/tests/wf/resreq_expr_float_v1_2.cwl +++ b/tests/wf/resreq_expr_float_v1_2.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.2 class: CommandLineTool requirements: diff --git a/tests/wf/scatter2.cwl b/tests/wf/scatter2.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/scatter2_subwf.cwl b/tests/wf/scatter2_subwf.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/schemadef-bug-1473.cwl b/tests/wf/schemadef-bug-1473.cwl old mode 100644 new mode 100755 index ad87ae08e..beab3211d --- a/tests/wf/schemadef-bug-1473.cwl +++ b/tests/wf/schemadef-bug-1473.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner { "$graph": [ { diff --git a/tests/wf/schemadef-tool-12.cwl b/tests/wf/schemadef-tool-12.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/schemadef-tool.cwl b/tests/wf/schemadef-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/sec-tool.cwl b/tests/wf/sec-tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/sec-wf-out.cwl b/tests/wf/sec-wf-out.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/sec-wf.cwl b/tests/wf/sec-wf.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/secret_job.cwl b/tests/wf/secret_job.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/secret_wf.cwl b/tests/wf/secret_wf.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/separate_without_prefix.cwl b/tests/wf/separate_without_prefix.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/shm_size.cwl b/tests/wf/shm_size.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/storage_float.cwl b/tests/wf/storage_float.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/tar-param.cwl b/tests/wf/tar-param.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/three_step_color.cwl b/tests/wf/three_step_color.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/timelimit-fail.cwl b/tests/wf/timelimit-fail.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/timelimit.cwl b/tests/wf/timelimit.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/touch_tool.cwl b/tests/wf/touch_tool.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/trick_defaults.cwl b/tests/wf/trick_defaults.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/trick_defaults2.cwl b/tests/wf/trick_defaults2.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/vf-concat.cwl b/tests/wf/vf-concat.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/workreuse-fail.cwl b/tests/wf/workreuse-fail.cwl old mode 100644 new mode 100755 diff --git a/tests/wf/workreuse.cwl b/tests/wf/workreuse.cwl old mode 100644 new mode 100755 diff --git a/tests/with_doc.cwl b/tests/with_doc.cwl old mode 100644 new mode 100755 diff --git a/tests/without_doc.cwl b/tests/without_doc.cwl old mode 100644 new mode 100755 diff --git a/tox.ini b/tox.ini index c75d0cc47..c5740a845 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = - py3{8,9,10,11,12,13}-lint - py3{8,9,10,11,12,13}-unit - py3{8,9,10,11,12,13}-bandit - py3{8,9,10,11,12,13}-mypy + py3{9,10,11,12,13}-lint + py3{9,10,11,12,13}-unit + py3{9,10,11,12,13}-bandit + py3{9,10,11,12,13}-mypy py312-lintreadme py312-shellcheck py312-pydocstyle @@ -16,7 +16,6 @@ testpaths = tests [gh-actions] python = - 3.8: py38 3.9: py39 3.10: py310 3.11: py311 @@ -25,13 +24,13 @@ python = [testenv] skipsdist = - py3{8,9,10,11,12,13}-!{unit,mypy,lintreadme} = True + py3{9,10,11,12,13}-!{unit,mypy,lintreadme} = True description = - py3{8,9,10,11,12,13}-unit: Run the unit tests - py3{8,9,10,11,12,13}-lint: Lint the Python code - py3{8,9,10,11,12,13}-bandit: Search for common security issues - py3{8,9,10,11,12,13}-mypy: Check for type safety + py3{9,10,11,12,13}-unit: Run the unit tests + py3{9,10,11,12,13}-lint: Lint the Python code + py3{9,10,11,12,13}-bandit: Search for common security issues + py3{9,10,11,12,13}-mypy: Check for type safety py312-pydocstyle: docstring style checker py312-shellcheck: syntax check for shell scripts py312-lintreadme: Lint the README.rst→.md conversion @@ -44,14 +43,14 @@ passenv = SINGULARITY_FAKEROOT extras = - py3{8,9,10,11,12,13}-unit: deps + py3{9,10,11,12,13}-unit: deps deps = - py3{8,9,10,11,12,13}-{unit,lint,bandit,mypy}: -rrequirements.txt - py3{8,9,10,11,12,13}-{unit,mypy}: -rtest-requirements.txt - py3{8,9,10,11,12,13}-lint: -rlint-requirements.txt - py3{8,9,10,11,12,13}-bandit: bandit - py3{8,9,10,11,12,13}-mypy: -rmypy-requirements.txt + py3{9,10,11,12,13}-{unit,lint,bandit,mypy}: -rrequirements.txt + py3{9,10,11,12,13}-{unit,mypy}: -rtest-requirements.txt + py3{9,10,11,12,13}-lint: -rlint-requirements.txt + py3{9,10,11,12,13}-bandit: bandit + py3{9,10,11,12,13}-mypy: -rmypy-requirements.txt py312-pydocstyle: pydocstyle py312-pydocstyle: diff-cover py312-lintreadme: twine @@ -63,20 +62,19 @@ setenv = HOME = {envtmpdir} commands_pre = - py3{8,9,10,11,12,13}-unit: python -m pip install -U pip setuptools wheel + py3{9,10,11,12,13}-unit: python -m pip install -U pip setuptools wheel py312-lintreadme: python -m build --outdir {distdir} commands = - py3{8,9,10,11,12,13}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} - py3{8,9,10,11,12,13}-bandit: bandit -r cwltool - py3{8,9,10,11,12,13}-lint: make flake8 format-check codespell-check - py3{8,9,10,11,12,13}-mypy: make mypy PYTEST_EXTRA={posargs} - py3{8,9,10,11,12}-mypy: make mypyc PYTEST_EXTRA={posargs} + py3{9,10,11,12,13}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} + py3{9,10,11,12,13}-bandit: bandit -r cwltool + py3{9,10,11,12,13}-lint: make flake8 format-check codespell-check + py3{9,10,11,12,13}-mypy: make mypy mypyc PYTEST_EXTRA={posargs} py312-shellcheck: make shellcheck py312-pydocstyle: make diff_pydocstyle_report py312-lintreadme: twine check {distdir}/* skip_install = - py3{8,9,10,11,12,13}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true + py3{9,10,11,12,13}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true allowlist_externals = make