D->M

.coveragerc

@@ -0,0 +1,3 @@
+[coverage:report]
+omit =
+    */test_*

.github/codeql.yml

@@ -0,0 +1,52 @@
+name: "Code scanning - action"
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 19 * * 0'
+
+jobs:
+  CodeQL-Build:
+
+    # CodeQL runs on ubuntu-latest and windows-latest
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      # Override language selection by uncommenting this and choosing your languages
+      # with:
+      #   languages: go, javascript, csharp, python, cpp, java
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3

.github/dependabot.yml

@@ -0,0 +1,23 @@
+version: 2
+updates:
+
+  # Docker
+  - package-ecosystem: docker
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 25
+
+  # Python
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 25
+
+  # GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: ".github/workflows"
+    schedule:
+      interval: "monthly"
+    open-pull-requests-limit: 25

.github/workflows/manual-build.yml

@@ -0,0 +1,11 @@
+---
+name: Manual Build & Push
+on:
+ workflow_dispatch:
+jobs:
+  build-push:
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}-develop'
+      tags: br-${{ github.ref_name }}
+    secrets: inherit

.github/workflows/pr_build.yml

@@ -0,0 +1,43 @@
+---
+name: Pull Request Build, Tag, & Push
+on:
+  pull_request:
+    branches:
+      - develop
+      - main
+      - master
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - closed
+jobs:
+  build-develop-open:
+    if: github.base_ref == 'develop' && github.event.pull_request.merged == false
+    uses: kbase/.github/.github/workflows/reusable_build.yml@main
+    secrets: inherit
+  build-develop-merge:
+    if: github.base_ref == 'develop' && github.event.pull_request.merged == true
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}-develop'
+      tags: pr-${{ github.event.number }},latest
+    secrets: inherit
+  build-main-open:
+    if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == false
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}'
+      tags: pr-${{ github.event.number }}
+    secrets: inherit
+  build-main-merge:
+    if: (github.base_ref == 'main' || github.base_ref == 'master') && github.event.pull_request.merged == true
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}'
+      tags: pr-${{ github.event.number }},latest-rc
+    secrets: inherit
+  trivy-scans:
+    if: (github.base_ref == 'develop' || github.base_ref == 'main' || github.base_ref == 'master' ) && github.event.pull_request.merged == false
+    uses: kbase/.github/.github/workflows/reusable_trivy-scans.yml@main
+    secrets: inherit

.github/workflows/release-main.yml

@@ -0,0 +1,25 @@
+---
+name: Release - Build & Push Image
+on:
+  release:
+    branches:
+      - main
+      - master
+    types: [ published ]
+jobs:
+  check-source-branch:
+    uses: kbase/.github/.github/workflows/reusable_validate-branch.yml@main
+    with:
+      build_branch: '${{ github.event.release.target_commitish }}'
+  validate-release-tag:
+    needs: check-source-branch
+    uses: kbase/.github/.github/workflows/reusable_validate-release-tag.yml@main
+    with:
+      release_tag: '${{ github.event.release.tag_name }}'
+  build-push:
+    needs: validate-release-tag
+    uses: kbase/.github/.github/workflows/reusable_build-push.yml@main
+    with:
+      name: '${{ github.event.repository.name }}'
+      tags: '${{ github.event.release.tag_name }},latest'
+    secrets: inherit

.github/workflows/test.yml

@@ -0,0 +1,60 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Index Runner test
+
+on:
+  pull_request:
+    types:
+    - opened
+    - reopened
+    - synchronize
+    - ready_for_review
+  push:
+    # run workflow when merging to main or develop
+    branches:
+      - main
+      - master
+      - develop
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    env:
+      WORKSPACE_TOKEN: ${{ secrets.WORKSPACE_TOKEN }}
+      RE_API_TOKEN: ${{ secrets.RE_API_TOKEN }}
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.9.19
+
+    - name: Pip installation
+      run: python -m pip install --upgrade pip poetry
+
+    - name: Poetry installation
+      run: poetry install
+
+    - name: Run unit tests
+      run: scripts/run_unit_tests
+
+    - name: Run integration tests
+      run: |
+        echo "WORKSPACE_TOKEN=$WORKSPACE_TOKEN" > .env
+        echo "RE_API_TOKEN=$RE_API_TOKEN" >> .env
+        scripts/run_integration_tests
+
+    - name: Print Docker Compose logs
+      if: failure()
+      run: docker compose logs
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        fail_ci_if_error: true

CHANGELOG.md

@@ -6,8 +6,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.9.21] - 2022-08-28
+
+### Added
+- Configuration parameter to disable elastic indexing
+- Mimimal configuration file for an RE only indexer
+
+## [1.9.20] - 2022-06-15
+
+### Added
+- Configuration parameter for max objects in a workspace to skip narrative reindex
+- Configuration option to skip narrative reindex for certain workspaces
+- Configuration option for poll timeout
+- Configuration option to skip narrative reindexing
+
+## [1.9.19] - 2021-04-19
+
 ### Fixed
 - Fixing Samples Releng indexer to work with RE api time-travel.
+- Added prefix to default_search alias
+- Removed reads_1 from reads alias
+- Removed pangenome_orthologfamily from default_search
 
 ### Added
 - adding narrative_config.yaml spec file for separating narrative indexing.

Dockerfile

@@ -1,10 +1,11 @@
-FROM python:3.7-alpine
+FROM python:3.9.19-alpine
 
 # Dockerize related args
 ARG BUILD_DATE
 ARG VCS_REF
 ARG BRANCH=develop
 ENV DOCKERIZE_VERSION v0.6.1
+ENV CRYPTOGRAPHY_DONT_BUILD_RUST 1
 
 # Install dockerize
 RUN apk --update add --virtual deps curl tar gzip && \
@@ -24,14 +25,21 @@
 WORKDIR /app
 
 # Dependency for confluent-kafka
-RUN apk --update add librdkafka librdkafka-dev && ldconfig /usr/lib
+RUN apk --update add build-base git cmake bash
+RUN git clone --branch v2.6.1 https://github.com/confluentinc/librdkafka.git /tmp/librdkafka
+RUN cd /tmp/librdkafka && \
+    ./configure && \
+    make && \
+    make install && \
+    ldconfig /usr/lib
+RUN rm -rf /tmp/librdkafka
 
 # Install dependencies
 COPY pyproject.toml poetry.lock /app/
-RUN apk --update add --virtual deps python3-dev build-base libffi-dev libressl-dev && \
-    pip install --upgrade pip poetry==1.0.9 && \
+RUN apk --update add --virtual deps python3-dev build-base libffi-dev && \
+    pip install --upgrade pip poetry==2.1.2 && \
     poetry config virtualenvs.create false && \
-    poetry install --no-dev --no-interaction --no-ansi && \
+    poetry install --no-root --without dev --no-interaction --no-ansi && \
     apk del deps
 
 # Make the admin tools executable

README.md

@@ -10,13 +10,27 @@ sure to partition the topic to at least the number of running workers.
 
 ## Development
 
+### Dependencies
+
+Note that `docker-compose` 2.3+ is required to run the tests.
+
+In this project, python packages and dependencies are managed using poetry. To
+install the development dependencies, create and activate a python virtual
+environment and then run the `poetry install` command.
+
+### Running the tests
+
 Start the servers:
 
 ```sh
 docker-compose up
 ```
 
-Run the tests (servers need not be running, and will be shut down if they are):
+To run all the tests the `RE_API_TOKEN` and `WORKSPACE_TOKEN` environment
+variables must be set to admin tokens for the relation engine and workspace,
+respectively. This project is set up to read a `.env` file, so they may be set
+there. The following command will run the tests (servers need not be running,
+and will be shut down if they are):
 
 ```sh
 scripts/run_tests
@@ -45,6 +59,8 @@ You can set the following env vars:
 * `SKIP_RELENG` - skip imports into the relation engine (ArangoDB)
 * `SKIP_FEATURES` - skip any importing or indexing of genome features
 * `SKIP_INDICES` - comma-separated list of index names that the service will not write into.
+* `SKIP_WORKSPACES` - comma-separated list of workspaces that should be skipped
+* `MAX_OBJECT_REINDEX` - If the number of objects in the workspace are greater than this value, then the narrative object will not automatically be reindexed.
 * `ELASTICSEARCH_HOST` - host name of the elasticsearch server to use (do not prepend protocol)
 * `ELASTICSEARCH_PORT` - port to use for the elasticsearch server
 * `KBASE_ENDPOINT` - URL of kbase API services (default is "https://ci.kbase.us/services")
@@ -80,7 +96,7 @@ Examples:
 
 Show command help: `indexer_admin -h`
 
-_Reindex a specific object_ 
+_Reindex a specific object_
 
 ```sh
 # Reindex only if the doc does not exist
@@ -143,7 +159,7 @@ First, increment the versions found in `VERSION` and in `pyproject.toml`.
 
 Then, build the image and push to docker hub with `scripts/docker_deploy`.
 
-### Project anatomy
+### Project Anatomy
 
 * The main process and entrypoint for the app lives in `./src/index_runner/main.py`
 * The entrypoint for ES indexing is in `./src/index_runner/es_indexer.py`

VERSION

@@ -1 +1 @@
-1.9.18
+1.9.21