Merge branch 'dev' into topic/linter-overhaul

.ci/flake8_ignorelist.txt

@@ -1,7 +1,6 @@
 .git
 .tox
-.venv
-.venv3
+.venv*
 packages/*/.venv
 packages/*/build
 packages/*/dist

.ci/ignore-spelling.txt

@@ -1,2 +1,3 @@
 hda
 implementors
+purgable

.flake8

@@ -4,7 +4,8 @@
 # E203 is whitespace before ':'; we follow black's formatting here. See https://black.readthedocs.io/en/stable/faq.html#why-are-flake8-s-e203-and-w503-violated
 # E402 module level import not at top of file # TODO, we would like to improve this.
 # E501 is line length (delegated to black)
+# E701,E704 are multiple statements on one line; we follow black's formatting here. See https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#configuration
 # W503 is line breaks before binary operators, which has been reversed in PEP 8.
 # D** are docstring linting - which we mostly ignore except D302. (Hopefully we will solve more over time).
-ignore = B008,E203,E402,E501,W503,D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D204,D205,D206,D207,D208,D209,D210,D211,D300,D301,D400,D401,D402,D403,D412,D413
+ignore = B008,E203,E402,E501,E701,E704,W503,D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D204,D205,D206,D207,D208,D209,D210,D211,D300,D301,D400,D401,D402,D403,D412,D413
 exclude = lib/tool_shed/test/test_data/repos

.github/workflows/build_client.yaml

@@ -0,0 +1,36 @@
+name: Build client for selenium tests
+on:
+  workflow_call:
+    outputs:
+      commit-id:
+        description: Commit ID
+        value: ${{ jobs.build-client.outputs.commit-id }}
+jobs:
+  build-client:
+    runs-on: ubuntu-latest
+    outputs:
+      commit-id: ${{ steps.client-commit.outputs.commit }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
+      - name: get client commit
+        id: client-commit
+        shell: bash
+        run: echo "commit=$(git rev-parse HEAD 2>/dev/null)" >> $GITHUB_OUTPUT
+        working-directory: 'galaxy root'
+      - name: cache client build
+        uses: actions/cache@v3
+        id: cache
+        with:
+          key: galaxy-static-${{ steps.client-commit.outputs.commit }}
+          path: 'galaxy root/static'
+      - name: Build client
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: make client
+        working-directory: 'galaxy root'

.github/workflows/docs.yaml

@@ -62,7 +62,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
           case "$TARGET_BRANCH" in
-              release_[[:digit:]][[:digit:]].[[:digit:]][[:digit:]]|master)
+              release_[[:digit:]][[:digit:]].[[:digit:]][[:digit:]] | release_[[:digit:]][[:digit:]].[[:digit:]] | master)
                   UPLOAD_DIR=$TARGET_BRANCH
                   ;;
               dev)

.github/workflows/first_startup.yaml

@@ -14,14 +14,16 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 jobs:
-
+  build-client:
+    uses: ./.github/workflows/build_client.yaml
   test:
     name: Startup test
+    needs: build-client
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.11']
+        python-version: ['3.8', '3.12']
     defaults:
       run:
         shell: bash -l {0}
@@ -53,6 +55,11 @@ jobs:
         with:
           path: .tox
           key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-first-startup
+      - name: Restore client cache
+        uses: actions/cache@v3
+        with:
+          key: galaxy-static-${{ needs.build-client.outputs.commit-id }}
+          path: 'galaxy root/static'
       - name: Install tox
         run: pip install tox
       - name: run tests

.github/workflows/integration_selenium.yaml

@@ -23,9 +23,11 @@ concurrency:
 jobs:
   setup-selenium:
     uses: ./.github/workflows/setup_selenium.yaml
+  build-client:
+    uses: ./.github/workflows/build_client.yaml
   test:
     name: Test
-    needs: setup-selenium
+    needs: [setup-selenium, build-client]
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -71,6 +73,11 @@ jobs:
           node-version: '18.12.1'
           cache: 'yarn'
           cache-dependency-path: 'galaxy root/client/yarn.lock'
+      - name: Restore client cache
+        uses: actions/cache@v3
+        with:
+          key: galaxy-static-${{ needs.build-client.outputs.commit-id }}
+          path: 'galaxy root/static'
       - name: Run tests
         run: ./run_tests.sh --coverage -integration test/integration_selenium
         working-directory: 'galaxy root'

.github/workflows/lint.yaml

@@ -22,7 +22,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.11']
+        python-version: ['3.8', '3.12']
     env:
       LINT_PATH: 'lib/galaxy/dependencies/pinned-lint-requirements.txt'
       TYPE_PATH: 'lib/galaxy/dependencies/pinned-typecheck-requirements.txt'
@@ -55,6 +55,4 @@ jobs:
       - name: Run mypy checks
         run: tox -e mypy
       - uses: psf/black@stable
-        with:
-          version: "23.3.0"
       - uses: isort/isort-action@master

.github/workflows/lint_openapi_schema.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.11']
+        python-version: ['3.8', '3.12']
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/maintenance_bot.yaml

@@ -8,7 +8,7 @@ jobs:
     if: github.repository_owner == 'galaxyproject'
     runs-on: ubuntu-latest
     env:
-      MILESTONE_NUMBER: 25
+      MILESTONE_NUMBER: 26
     steps:
       - name: Get latest pull request labels
         id: get_pr_labels

.github/workflows/osx_startup.yaml

@@ -12,13 +12,16 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 jobs:
+  build-client:
+    uses: ./.github/workflows/build_client.yaml
   test:
     name: Startup test
     runs-on: macos-latest
+    needs: build-client
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.11']
+        python-version: ['3.8', '3.12']
     defaults:
       run:
         shell: bash -l {0}
@@ -50,6 +53,11 @@ jobs:
         uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: ''
+      - name: Restore client cache
+        uses: actions/cache@v3
+        with:
+          key: galaxy-static-${{ needs.build-client.outputs.commit-id }}
+          path: 'galaxy root/static'
       - name: Install tox
         run: pip install tox
       - name: Run tests

.github/workflows/reports_startup.yaml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.11']
+        python-version: ['3.8', '3.12']
     defaults:
       run:
         shell: bash -l {0}

.github/workflows/selenium.yaml

@@ -23,9 +23,11 @@ concurrency:
 jobs:
   setup-selenium:
     uses: ./.github/workflows/setup_selenium.yaml
+  build-client:
+    uses: ./.github/workflows/build_client.yaml
   test:
     name: Test
-    needs: setup-selenium
+    needs: [setup-selenium, build-client]
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -71,6 +73,11 @@ jobs:
         with:
           path: 'galaxy root/.venv'
           key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-selenium
+      - name: Restore client cache
+        uses: actions/cache@v3
+        with:
+          key: galaxy-static-${{ needs.build-client.outputs.commit-id }}
+          path: 'galaxy root/static'
       - name: Run tests
         run: ./run_tests.sh --coverage -selenium lib/galaxy_test/selenium -- --num-shards=3 --shard-id=${{ matrix.chunk }}
         working-directory: 'galaxy root'

.github/workflows/test_galaxy_packages.yaml

@@ -18,11 +18,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.11']
+        python-version: ['3.8', '3.12']
     steps:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}

.github/workflows/toolshed.yaml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.11']
+        python-version: ['3.8', '3.12']
         shed-api: ['v1', 'v2']
         test-install-client: ['galaxy_api', 'standalone']
     services:

.github/workflows/unit.yaml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.11']
+        python-version: ['3.8', '3.12']
     steps:
       - uses: actions/checkout@v3
         with:

.gitignore

@@ -7,8 +7,7 @@ scripts/scramble/lib
 scripts/scramble/archives
 
 # Python virtualenv
-.venv
-.venv3
+.venv*/
 
 # Python build artifacts
 build

.pre-commit-config.yaml.sample

@@ -1,10 +1,10 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 24.1.1
     hooks:
     - id: black
   - repo: https://github.com/pycqa/flake8
-    rev: 5.0.4
+    rev: 7.0.0
     hooks:
     - id: flake8
   - repo: https://github.com/pre-commit/mirrors-prettier
@@ -16,7 +16,7 @@ repos:
         additional_dependencies:
           - [email protected] # Workaround. See https://github.com/pre-commit/mirrors-prettier/issues/29
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0  # Use the ref you want to point at
+    rev: v4.5.0  # Use the ref you want to point at
     hooks:
       - id: trailing-whitespace
       - id: check-merge-conflict
@@ -30,7 +30,7 @@ repos:
     - id: shell-lint
       args: [--format=json]
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.19.2
+    rev: 0.27.4
     hooks:
       - id: check-github-workflows
   - repo: local

.redocly.lint-ignore.yaml

@@ -12,8 +12,43 @@ _schema.yaml:
       #/paths/~1api~1histories~1{history_id}~1contents~1{history_content_id}~1extra_files
     - >-
       #/paths/~1api~1histories~1{history_id}~1contents~1{history_content_id}~1metadata_file
+    - >-
+      #/paths/~1api~1histories~1{history_id}~1contents~1{history_content_id}~1tags
     - '#/paths/~1api~1histories~1{history_id}~1contents~1{id}~1validate'
     - '#/paths/~1api~1histories~1{history_id}~1contents~1{type}s~1{id}'
+    - '#/paths/~1api~1invocations~1{invocation_id}~1biocompute'
+    - '#/paths/~1api~1invocations~1{invocation_id}~1jobs_summary'
+    - '#/paths/~1api~1invocations~1{invocation_id}~1prepare_store_download'
+    - '#/paths/~1api~1invocations~1{invocation_id}~1report'
+    - '#/paths/~1api~1invocations~1{invocation_id}~1report.pdf'
+    - '#/paths/~1api~1invocations~1{invocation_id}~1step_jobs_summary'
+    - '#/paths/~1api~1invocations~1{invocation_id}~1write_store'
+    - '#/paths/~1api~1quotas~1{id}~1purge'
+    - '#/paths/~1api~1users~1{user_id}~1api_key'
+    - '#/paths/~1api~1users~1{user_id}~1beacon'
+    - '#/paths/~1api~1users~1{user_id}~1custom_builds'
+    - '#/paths/~1api~1users~1{user_id}~1recalculate_disk_usage'
+    - '#/paths/~1api~1users~1{user_id}~1send_activation_email'
+    - '#/paths/~1api~1users~1{user_id}~1usage'
+  no-unused-components:
+    - '#/components/schemas/FetchDataPayload'
+  security-defined:
+    - '#/paths/~1api~1authenticate~1baseauth/get'
 _shed_schema.yaml:
   no-empty-servers:
     - '#/openapi'
+  security-defined:
+    - '#/paths/~1api~1authenticate~1baseauth/get'
+    - '#/paths/~1api~1categories~1{encoded_category_id}/get'
+    - '#/paths/~1api~1ga4gh~1trs~1v2~1service-info/get'
+    - '#/paths/~1api~1ga4gh~1trs~1v2~1toolClasses/get'
+    - '#/paths/~1api~1ga4gh~1trs~1v2~1tools/get'
+    - '#/paths/~1api~1repositories~1get_ordered_installable_revisions/get'
+    - '#/paths/~1api~1repositories~1updates/get'
+    - '#/paths/~1api~1repositories~1{encoded_repository_id}/get'
+    - '#/paths/~1api~1repositories~1{encoded_repository_id}~1metadata/get'
+    - >-
+      #/paths/~1api~1repositories~1{encoded_repository_id}~1revisions~1{changeset_revision}~1readmes/get
+    - '#/paths/~1api~1version/get'
+    - >-
+      #/paths/~1api_internal~1repositories~1{encoded_repository_id}~1metadata/get

CONTRIBUTING.md

@@ -1,24 +1,24 @@
 # Contributing
 
 Galaxy welcomes new development!  This document briefly describes how to
-contribute to the [core galaxy
-repository](https://github.com/galaxyproject/galaxy).
-For information on contributing more broadly to the Galaxy ecosystem and a
-deeper discussion of some of these points - please see the
-[Develop](https://galaxyproject.org/develop) section of the [Galaxy Community
-Hub](https://galaxyproject.org) and the [Galaxy Code
-Architecture](https://training.galaxyproject.org/training-material/topics/dev/tutorials/architecture/slides.html#1) slides that are part of the [Galaxy Training
-Materials](https://training.galaxyproject.org/).
+contribute to the [core galaxy repository](https://github.com/galaxyproject/galaxy).
+For general information on the Galaxy ecosystem, please see the
+[Galaxy Community Hub](https://galaxyproject.org).
+For a description of how the Galaxy code is structured, see the
+[Galaxy Code Architecture slides](https://training.galaxyproject.org/training-material/topics/dev/tutorials/architecture/slides.html)
+that are part of the [Galaxy Training Materials](https://training.galaxyproject.org/).
 
 ## Before you Begin
 
 If you have an idea for a feature to add or an approach for a bugfix, it is
 best to communicate with Galaxy developers early. The primary venue for this is
 the [GitHub issue tracker](https://github.com/galaxyproject/galaxy/issues).
 Browse through existing GitHub issues and if one seems related, comment on it.
-For more direct communication, Galaxy developers are generally available via
-[Gitter](https://gitter.im/galaxyproject/dev) and on the [development mailing
-list](http://dev.list.galaxyproject.org).
+For more direct communication, Galaxy developers are generally available on
+the [Galaxy Matrix space](https://matrix.to/#/#galaxyproject:matrix.org), in
+particular on the [galaxyprojec/dev channel](https://matrix.to/#/#galaxyproject_dev:gitter.im)
+and in the various [Working Group](https://galaxyproject.org/community/wg/)
+channels.
 
 If you're looking to help but aren't sure where to start, we also maintain a
 [tag](https://github.com/galaxyproject/galaxy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22)

CONTRIBUTORS.md

@@ -69,7 +69,7 @@ The following individuals have contributed code to Galaxy:
 * Ben Fulton <[email protected]>
 * Carrie Ganote <[email protected]>
 * Ryan Golhar <[email protected]>
-* Jeremy Goecks <jeremy.goecks@emory.edu> <[email protected]>
+* Jeremy Goecks <jeremy.goecks@moffitt.org> 
 * Nuwan Goonasekera <[email protected]>
 * Björn Grüning <[email protected]> <[email protected]>
 * Aysam Guerler <[email protected]>

Makefile

@@ -2,7 +2,7 @@
 VENV?=.venv
 # Source virtualenv to execute command (darker, sphinx, twine, etc...)
 IN_VENV=if [ -f "$(VENV)/bin/activate" ]; then . "$(VENV)/bin/activate"; fi;
-RELEASE_CURR:=23.3
+RELEASE_CURR:=24.0
 RELEASE_UPSTREAM:=upstream
 CONFIG_MANAGE=$(IN_VENV) python lib/galaxy/config/config_manage.py
 PROJECT_URL?=https://github.com/galaxyproject/galaxy