PTV-1888

.devcontainer/devcontainer.json

@@ -0,0 +1,40 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
+{
+	"name": "staging service devcontainer",
+	"dockerComposeFile": [
+		"docker-compose.yml"
+	],
+	// The 'service' property is the name of the service for the container that VS Code should
+	// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
+	"service": "staging_service",
+	// "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached",
+	// The optional 'workspaceFolder' property is the path VS Code should open by default when
+	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
+	"workspaceFolder": "/workspace",
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.python",
+				"tamasfe.even-better-toml",
+				"mhutchie.git-graph",
+				"ms-python.black-formatter",
+				"ms-python.flake8",
+				"mikoz.autoflake-extension",
+				"ms-azuretools.vscode-docker",
+				"DavidAnson.vscode-markdownlint",
+				"matangover.mypy"
+			]
+		}
+	}
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Uncomment the next line to run commands after the container is created.
+	// "postCreateCommand": "cat /etc/os-release",
+	// Configure tool-specific properties.
+	// "customizations": {},
+	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "devcontainer"
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,29 @@
+version: '3.6'
+networks:
+  kbase-dev:
+    name: kbase-dev
+services:
+  staging_service:
+    build:
+      context: ..
+      target: dev
+
+    # image: kbase/staging_server_devcontainer:dev
+
+    container_name: staging_server_devcontainer
+    dns: 8.8.8.8
+    volumes:
+      - ..:/workspace
+    networks:
+      - kbase-dev
+    # Set some default environment variables, which enable the
+    # service to work inside the container without too much fussing.
+    env:
+      KB_DEPLOYMENT_CONFIG=./deployment/conf/local.cfg
+      FILE_LIFETIME=90
+    # Required for a devcontainer -- keeps the container running.
+    # Don't worry, our main interaction with the container is through the 
+    # VSC terminal, which for a devcontainer opens a shell within the
+    # container.
+    command: /bin/sh -c "while sleep 1000; do :; done"
+

.github/workflows/run_tests.yml

@@ -1,26 +1,12 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
 name: Run Staging Service Tests
-
 on:
   [pull_request]
-
 jobs:
   build:
-
     runs-on: ubuntu-latest
-
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.11
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.11.4
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: Test with pytest
-      run: |
-        bash run_tests.sh
+    - name: Checkout the repo
+      uses: actions/checkout@v3
+
+    - name: Run the tests
+      run: ./development/scripts/run scripts/run_tests.sh

.gitignore

@@ -107,4 +107,10 @@ data/
 .DS_Store
 .virtualenvs/
 test.env
-run_tests_single.sh
+run_tests_single.sh
+_attic
+
+# IDEs
+
+# jetbrains
+.idea

.markdownlint.yaml

@@ -0,0 +1,10 @@
+MD013:
+  # Matches black line length for python code.
+  line_length: 88
+  # some long examples would break the line length and are awkward to rearrange
+  code_blocks: false
+
+# We don't want the annoying warnings when there are duplicated non-sibling headers as
+# many types of docs have a pattern for headings that is repeated throughout the document.
+MD024: 
+  siblings_only: true

Dockerfile

@@ -1,34 +1,78 @@
-FROM python:3.11.4-slim-buster
+FROM python:3.11.5-slim-bookworm AS base
 # -----------------------------------------
-RUN mkdir -p /kb/deployment/lib
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends zip=3.0-11+b1 unzip=6.0-23+deb10u3 bzip2=1.0.6-9.2~deb10u2 libmagic-dev=1:5.35-4+deb10u2 htop=2.2.0-1+b1 wget=1.20.1-1.1 && \
+    apt-get install -y --no-install-recommends zip=3.0-13 unzip=6.0-28 bzip2=1.0.8-5+b1 libmagic-dev=1:5.44-3 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-COPY ./requirements.txt /requirements.txt
-RUN python -m pip install "pip==23.1.2" && pip install -r /requirements.txt && rm /requirements.txt
+COPY requirements.txt /kb/requirements.txt
+RUN python -m pip install "pip==23.2.1" && pip install -r /kb/requirements.txt && rm -r /kb
 
+# The BUILD_DATE value seem to bust the docker cache when the timestamp changes, move to
+# the end
+LABEL org.label-schema.build-date=$BUILD_DATE \
+    org.label-schema.vcs-url="https://github.com/kbase/staging_service.git" \
+    org.label-schema.vcs-ref=$VCS_REF \
+    org.label-schema.schema-version="1.1.8" \
+    us.kbase.vcs-branch=$BRANCH \
+    maintainer="Steve Chan [email protected]"
+
+#
+# Dev Layer 
+# Used in devcontainer, and as base for tools
+#
+FROM base AS dev
+# Install OS dependencies required by or nice-to-have in a development image
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends htop=3.2.2-2 wget=1.21.3-1+b2 git=1:2.39.2-1.1 openssh-client=1:9.2p1-2 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+# Install Python dependencies require by development tools (cli tools and devcontainer)
+COPY ./requirements_dev.txt /kb/requirements_dev.txt
+RUN pip install -r /kb/requirements_dev.txt && rm -r /kb
+WORKDIR /kb/module
+# Note - entrypoint defined in docker compose file, and /kb/module is volume mounted by
+# the devcontainer and the tools
+
+#
+# Prod layer
+#
+FROM base AS prod
+
+# Install globus configuration into expected location.
+# TODO: point to location for documentation of this.
 COPY ./globus.cfg /etc/globus.cfg
 RUN touch /var/log/globus.log && chmod 777 /var/log/globus.log
 
+# We expect this to run on port 3000
+# TODO: this is weird, kbase services usually run at port 5000.
+EXPOSE 3000
+
+# We keep the entire repo in /kb/module; for why, I know not.
+# TODO: could someone add a comment here explaining why?
 COPY ./ /kb/module
-RUN cp -r /kb/module/staging_service /kb/deployment/lib
-RUN cp -r /kb/module/deployment /kb
+WORKDIR /kb/module
 
+# Otherwise, the service is installed in /kb/deployment (!)
+# RUN mkdir -p /kb/deployment/lib
+# RUN cp -r /kb/module/staging_service /kb/deployment/lib
 
-EXPOSE 3000
+#
+# Here we copy all of the required runtime components that need 
+# to be in the image.
+#
 
-WORKDIR /kb/deployment/lib
+# This contains the entrypoint
+COPY ./deployment/bin /kb/deployment/bin
 
+# This contains the CI deployment 
+# TODO: why is it copied to the codebase, though?
+COPY ./deployment/conf/deployment.cfg /kb/deployment/conf/deployment.cfg
 
-# The BUILD_DATE value seem to bust the docker cache when the timestamp changes, move to
-# the end
-LABEL org.label-schema.build-date=$BUILD_DATE \
-    org.label-schema.vcs-url="https://github.com/kbase/staging_service.git" \
-    org.label-schema.vcs-ref=$VCS_REF \
-    org.label-schema.schema-version="1.1.8" \
-    us.kbase.vcs-branch=$BRANCH \
-    maintainer="Steve Chan [email protected]"
+# Configuration for mapping file extensions to importers
+COPY ./deployment/conf/supported_apps_w_extensions.json /kb/deployment/conf/supported_apps_w_extensions.json
+
+# The service code.
+COPY ./staging_service /kb/deployment/lib/staging_service
 
-ENTRYPOINT ["/kb/deployment/bin/entrypoint.sh"]
+ENTRYPOINT ["/kb/module/deployment/bin/entrypoint.sh"]