From ea3cc368b21031cef10b0c9e380605f188e0f557 Mon Sep 17 00:00:00 2001 From: John Bates Date: Thu, 20 Feb 2025 23:10:45 +0000 Subject: [PATCH 01/10] [DO NOT MERGE] Testing GATK installed on Jupyter image --- src/gatk-demo/devcontainer-template.json | 33 ++++++++++++++++++++++++ src/gatk-demo/docker-compose.yaml | 25 ++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 src/gatk-demo/devcontainer-template.json create mode 100644 src/gatk-demo/docker-compose.yaml diff --git a/src/gatk-demo/devcontainer-template.json b/src/gatk-demo/devcontainer-template.json new file mode 100644 index 00000000..fa709676 --- /dev/null +++ b/src/gatk-demo/devcontainer-template.json @@ -0,0 +1,33 @@ +{ + "id": "gatk-demo-template", + "version": "0.0.1", + "name": "GATK on Jupyter Demo", + "description": "A Template to run Jupyter lab with GATK installed", + "documentationURL": "https://github.com/verily-src/workbench-app-devcontainers/tree/master/src/gatk-demo-template", + "licenseURL": "https://github.com/verily-src/workbench-app-devcontainers/blob/master/LICENSE", + "options": { + "cloud": { + "type": "string", + "description": "VM cloud environment", + "proposals": ["gcp", "aws"], + "default": "gcp" + }, + "login": { + "type": "string", + "description": "Whether to log in to workbench CLI", + "proposals": ["true", "false"], + "default": "false" + }, + "containerImage": { + "type": "string", + "description": "The container image to use", + "default": "us-central1-docker.pkg.dev/wb-fulgent-broccoli-5942/gatk-demo-repo/test1:2025022" + }, + "containerPort": { + "type": "number", + "description": "The port to expose the container on", + "default": 8888 + } + }, + "platforms": ["Any"] +} diff --git a/src/gatk-demo/docker-compose.yaml b/src/gatk-demo/docker-compose.yaml new file mode 100644 index 00000000..48d74477 --- /dev/null +++ b/src/gatk-demo/docker-compose.yaml @@ -0,0 +1,25 @@ +version: "2.4" +services: + app: + container_name: "application-server" + image: "${templateOption:containerImage}" + user: "jupyter" + restart: always + volumes: + - .:/workspace:cached + ports: + - "${templateOption:containerPort}:${templateOption:containerPort}" + command: "/home/jupyter/.local/bin/jupyter lab" + environment: + SHELL: "/bin/bash" + networks: + - app-network + cap_add: + - SYS_ADMIN + devices: + - /dev/fuse + security_opt: + - apparmor:unconfined +networks: + app-network: + external: true From d8af49ea9e3ee064cc4ea68d68a7f23d3c897aa9 Mon Sep 17 00:00:00 2001 From: John Bates Date: Fri, 21 Feb 2025 18:03:20 +0000 Subject: [PATCH 02/10] Populated container image and ports directly in docker-compose.yaml --- src/gatk-demo/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gatk-demo/docker-compose.yaml b/src/gatk-demo/docker-compose.yaml index 48d74477..16f24076 100644 --- a/src/gatk-demo/docker-compose.yaml +++ b/src/gatk-demo/docker-compose.yaml @@ -2,13 +2,13 @@ version: "2.4" services: app: container_name: "application-server" - image: "${templateOption:containerImage}" + image: "us-central1-docker.pkg.dev/wb-fulgent-broccoli-5942/gatk-demo-repo/test1:2025022" user: "jupyter" restart: always volumes: - .:/workspace:cached ports: - - "${templateOption:containerPort}:${templateOption:containerPort}" + - "8888:8888" command: "/home/jupyter/.local/bin/jupyter lab" environment: SHELL: "/bin/bash" From 8cd778bd8081c51a2a15fddd460a6d839b5d60ef Mon Sep 17 00:00:00 2001 From: John Bates Date: Mon, 24 Feb 2025 23:14:39 +0000 Subject: [PATCH 03/10] Pointing to container image on dev. --- src/gatk-demo/devcontainer-template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gatk-demo/devcontainer-template.json b/src/gatk-demo/devcontainer-template.json index fa709676..cc7c4775 100644 --- a/src/gatk-demo/devcontainer-template.json +++ b/src/gatk-demo/devcontainer-template.json @@ -21,7 +21,7 @@ "containerImage": { "type": "string", "description": "The container image to use", - "default": "us-central1-docker.pkg.dev/wb-fulgent-broccoli-5942/gatk-demo-repo/test1:2025022" + "default": "us-central1-docker.pkg.dev/vwb-dev-sparkly-eggplant-5788/gatk-demo-repo/test1:20250224" }, "containerPort": { "type": "number", From af925cad33ef893d156c399b33cc56bc29a5c41f Mon Sep 17 00:00:00 2001 From: John Bates Date: Mon, 24 Feb 2025 23:21:31 +0000 Subject: [PATCH 04/10] Renamed devcontainer-template.json to devcontainer.json --- src/gatk-demo/{devcontainer-template.json => devcontainer.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/gatk-demo/{devcontainer-template.json => devcontainer.json} (100%) diff --git a/src/gatk-demo/devcontainer-template.json b/src/gatk-demo/devcontainer.json similarity index 100% rename from src/gatk-demo/devcontainer-template.json rename to src/gatk-demo/devcontainer.json From cc547043e6ebdb73fee87b23efb9d60b20a17cb8 Mon Sep 17 00:00:00 2001 From: John Bates Date: Mon, 24 Feb 2025 23:31:20 +0000 Subject: [PATCH 05/10] Renamed devcontainer.json .devcontainer.json --- src/gatk-demo/{devcontainer.json => .devcontainer.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/gatk-demo/{devcontainer.json => .devcontainer.json} (100%) diff --git a/src/gatk-demo/devcontainer.json b/src/gatk-demo/.devcontainer.json similarity index 100% rename from src/gatk-demo/devcontainer.json rename to src/gatk-demo/.devcontainer.json From 2e49429b9ac2507ee68f43c8614fb8d096896fe8 Mon Sep 17 00:00:00 2001 From: John Bates Date: Mon, 24 Feb 2025 23:56:34 +0000 Subject: [PATCH 06/10] Pointed dockercompose.yaml container image to the dev image. --- src/gatk-demo/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gatk-demo/docker-compose.yaml b/src/gatk-demo/docker-compose.yaml index 16f24076..d5666e50 100644 --- a/src/gatk-demo/docker-compose.yaml +++ b/src/gatk-demo/docker-compose.yaml @@ -2,7 +2,7 @@ version: "2.4" services: app: container_name: "application-server" - image: "us-central1-docker.pkg.dev/wb-fulgent-broccoli-5942/gatk-demo-repo/test1:2025022" + image: "us-central1-docker.pkg.dev/vwb-dev-sparkly-eggplant-5788/gatk-demo-repo/test1:20250224" user: "jupyter" restart: always volumes: From 306925cca52d5218d18a54c1853a305ede39ef3a Mon Sep 17 00:00:00 2001 From: John Bates Date: Tue, 25 Feb 2025 00:11:33 +0000 Subject: [PATCH 07/10] Total reset from ../custom-blah-blah --- src/gatk-demo/.devcontainer.json | 81 +++++++++++++++--------- src/gatk-demo/devcontainer-template.json | 33 ++++++++++ src/gatk-demo/docker-compose.yaml | 7 +- 3 files changed, 86 insertions(+), 35 deletions(-) create mode 100644 src/gatk-demo/devcontainer-template.json diff --git a/src/gatk-demo/.devcontainer.json b/src/gatk-demo/.devcontainer.json index cc7c4775..28560022 100644 --- a/src/gatk-demo/.devcontainer.json +++ b/src/gatk-demo/.devcontainer.json @@ -1,33 +1,54 @@ { - "id": "gatk-demo-template", - "version": "0.0.1", - "name": "GATK on Jupyter Demo", - "description": "A Template to run Jupyter lab with GATK installed", - "documentationURL": "https://github.com/verily-src/workbench-app-devcontainers/tree/master/src/gatk-demo-template", - "licenseURL": "https://github.com/verily-src/workbench-app-devcontainers/blob/master/LICENSE", - "options": { - "cloud": { - "type": "string", - "description": "VM cloud environment", - "proposals": ["gcp", "aws"], - "default": "gcp" - }, - "login": { - "type": "string", - "description": "Whether to log in to workbench CLI", - "proposals": ["true", "false"], - "default": "false" - }, - "containerImage": { - "type": "string", - "description": "The container image to use", - "default": "us-central1-docker.pkg.dev/vwb-dev-sparkly-eggplant-5788/gatk-demo-repo/test1:20250224" - }, - "containerPort": { - "type": "number", - "description": "The port to expose the container on", - "default": 8888 + "name": "JupyterLab with GATK installed", + "dockerComposeFile": "docker-compose.yaml", + "service": "app", + "shutdownAction": "none", + "workspaceFolder": "/workspace", + "postCreateCommand": [ + "./startupscript/post-startup.sh", + "jupyter", + "/home/jupyter", + "${templateOption:cloud}", + "${templateOption:login}" + ], + // re-mount bucket files on container start up + "postStartCommand": [ + "./startupscript/remount-on-restart.sh", + "jupyter", + "/home/jupyter", + "${templateOption:cloud}", + "${templateOption:login}" + ], + "remoteUser": "root", + "customizations": { + "workbench": { + "opens": { + "extensions": [ + // Source + ".ipynb", + ".R", + ".py", + // Documents + ".md", + ".html", + ".latex", + ".pdf", + // Images + ".bmp", + ".gif", + ".jpeg", + ".jpg", + ".png", + ".svg", + // Data + ".csv", + ".tsv", + ".json", + ".vl" + ], + "fileUrlSuffix": "/lab/tree/{path}", + "folderUrlSuffix": "/lab/tree/{path}" + } } - }, - "platforms": ["Any"] + } } diff --git a/src/gatk-demo/devcontainer-template.json b/src/gatk-demo/devcontainer-template.json new file mode 100644 index 00000000..c8584876 --- /dev/null +++ b/src/gatk-demo/devcontainer-template.json @@ -0,0 +1,33 @@ +{ + "id": "custom-workbench-jupyter-template", + "description": "A template used to serve the Workbench JupyterLab container image", + "version": "0.0.1", + "name": "Workbench Prebuilt JupyterLab Template", + "documentationURL": "https://github.com/verily-src/workbench-app-devcontainers/tree/master/src/custom-workbench-jupyter-template", + "licenseURL": "https://github.com/verily-src/workbench-app-devcontainers/blob/master/LICENSE", + "options": { + "cloud": { + "type": "string", + "description": "VM cloud environment", + "proposals": ["gcp", "aws"], + "default": "gcp" + }, + "login": { + "type": "string", + "description": "Whether to log in to workbench CLI", + "proposals": ["true", "false"], + "default": "false" + }, + "containerImage": { + "type": "string", + "description": "The container image to use", + "default": "us-central1-docker.pkg.dev/vwb-dev-sparkly-eggplant-5788/gatk-demo-repo/test1:20250224" + }, + "containerPort": { + "type": "number", + "description": "The port to expose the container on", + "default": 8888 + } + }, + "platforms": ["Any"] +} diff --git a/src/gatk-demo/docker-compose.yaml b/src/gatk-demo/docker-compose.yaml index d5666e50..685dfb06 100644 --- a/src/gatk-demo/docker-compose.yaml +++ b/src/gatk-demo/docker-compose.yaml @@ -2,16 +2,13 @@ version: "2.4" services: app: container_name: "application-server" - image: "us-central1-docker.pkg.dev/vwb-dev-sparkly-eggplant-5788/gatk-demo-repo/test1:20250224" + image: "${templateOption:containerImage}" user: "jupyter" restart: always volumes: - .:/workspace:cached ports: - - "8888:8888" - command: "/home/jupyter/.local/bin/jupyter lab" - environment: - SHELL: "/bin/bash" + - "${templateOption:containerPort}:${templateOption:containerPort}" networks: - app-network cap_add: From 6cf0713cd6cdde1a3d5b497b2c09dbf850966780 Mon Sep 17 00:00:00 2001 From: John Bates Date: Tue, 25 Feb 2025 00:33:40 +0000 Subject: [PATCH 08/10] hardcoded template options in dockercompose.yaml --- src/gatk-demo/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gatk-demo/docker-compose.yaml b/src/gatk-demo/docker-compose.yaml index 685dfb06..0c536531 100644 --- a/src/gatk-demo/docker-compose.yaml +++ b/src/gatk-demo/docker-compose.yaml @@ -2,13 +2,13 @@ version: "2.4" services: app: container_name: "application-server" - image: "${templateOption:containerImage}" + image: "us-central1-docker.pkg.dev/vwb-dev-sparkly-eggplant-5788/gatk-demo-repo/test1:20250224" user: "jupyter" restart: always volumes: - .:/workspace:cached ports: - - "${templateOption:containerPort}:${templateOption:containerPort}" + - "8888:8888" networks: - app-network cap_add: From a01394211a6f0cee215328269b8e1b3f543f86c0 Mon Sep 17 00:00:00 2001 From: John Bates Date: Tue, 25 Feb 2025 16:56:45 +0000 Subject: [PATCH 09/10] Point to the 20250225 container image --- src/gatk-demo/devcontainer-template.json | 2 +- src/gatk-demo/docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gatk-demo/devcontainer-template.json b/src/gatk-demo/devcontainer-template.json index c8584876..8a10ddff 100644 --- a/src/gatk-demo/devcontainer-template.json +++ b/src/gatk-demo/devcontainer-template.json @@ -21,7 +21,7 @@ "containerImage": { "type": "string", "description": "The container image to use", - "default": "us-central1-docker.pkg.dev/vwb-dev-sparkly-eggplant-5788/gatk-demo-repo/test1:20250224" + "default": "us-central1-docker.pkg.dev/vwb-dev-sparkly-eggplant-5788/gatk-demo-repo/test1:20250225" }, "containerPort": { "type": "number", diff --git a/src/gatk-demo/docker-compose.yaml b/src/gatk-demo/docker-compose.yaml index 0c536531..f5209f88 100644 --- a/src/gatk-demo/docker-compose.yaml +++ b/src/gatk-demo/docker-compose.yaml @@ -2,7 +2,7 @@ version: "2.4" services: app: container_name: "application-server" - image: "us-central1-docker.pkg.dev/vwb-dev-sparkly-eggplant-5788/gatk-demo-repo/test1:20250224" + image: "us-central1-docker.pkg.dev/vwb-dev-sparkly-eggplant-5788/gatk-demo-repo/test1:20250225" user: "jupyter" restart: always volumes: From d435b2716e14c0fa4e7ad41d9c61e429a9db23a9 Mon Sep 17 00:00:00 2001 From: John Bates Date: Tue, 25 Feb 2025 18:03:26 +0000 Subject: [PATCH 10/10] Point to container in prod artifact registry --- src/gatk-demo/devcontainer-template.json | 2 +- src/gatk-demo/docker-compose.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gatk-demo/devcontainer-template.json b/src/gatk-demo/devcontainer-template.json index 8a10ddff..1f937845 100644 --- a/src/gatk-demo/devcontainer-template.json +++ b/src/gatk-demo/devcontainer-template.json @@ -21,7 +21,7 @@ "containerImage": { "type": "string", "description": "The container image to use", - "default": "us-central1-docker.pkg.dev/vwb-dev-sparkly-eggplant-5788/gatk-demo-repo/test1:20250225" + "default": "us-central1-docker.pkg.dev/wb-fulgent-broccoli-5942/gatk-demo-repo/test1:20250225" }, "containerPort": { "type": "number", diff --git a/src/gatk-demo/docker-compose.yaml b/src/gatk-demo/docker-compose.yaml index f5209f88..3bbea2c5 100644 --- a/src/gatk-demo/docker-compose.yaml +++ b/src/gatk-demo/docker-compose.yaml @@ -2,7 +2,7 @@ version: "2.4" services: app: container_name: "application-server" - image: "us-central1-docker.pkg.dev/vwb-dev-sparkly-eggplant-5788/gatk-demo-repo/test1:20250225" + image: "us-central1-docker.pkg.dev/wb-fulgent-broccoli-5942/gatk-demo-repo/test1:20250225" user: "jupyter" restart: always volumes: