Dev netpyne updates

.github/workflows/docker-jupyterlab.yml

@@ -1,4 +1,4 @@
-name: Docker Image Build Jupyterlab
+name: Docker Image Build JupyterLab
 
 on:
   push:
@@ -13,17 +13,18 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Build the Docker image
       run: |
         cd applications/jupyterlab
-        docker build -t myjlab -f Dockerfile --no-cache .
+        ./build_local.sh 
 
     - name: Info on Docker image sizes
       run: |
         docker images
 
     - name: Run the Docker container and list python installs
       run: |
-        docker run -t --rm --entrypoint /bin/bash myjlab  -c "pip3 list"
+        cd applications/jupyterlab
+        ./pip_omv_info.sh

.gitignore

@@ -14,3 +14,4 @@ skaffold-overrides.yaml
 /libraries/client/netpyne-web/
 /.venv
 /libraries/client/*.pyc
+/applications/netpyne/shared

applications/accounts/deploy/resources/realm.json

@@ -158,63 +158,6 @@
                 ]
             },
         "clients": [
-             {
-            "id": "18893fbb-8252-4aaa-bc9b-60799ceb9932",
-            "clientId": "account-console",
-            "name": "${client_account-console}",
-            "rootUrl": "${authBaseUrl}",
-            "baseUrl": {{ printf "/realms/%s/account" .Values.namespace | quote }},
-            "surrogateAuthRequired": false,
-            "enabled": true,
-            "alwaysDisplayInConsole": false,
-            "clientAuthenticatorType": "client-secret",
-            "redirectUris": [
-                {{ printf "/realms/%s/account/*" .Values.namespace | quote }}
-            ],
-            "webOrigins": [],
-            "notBefore": 0,
-            "bearerOnly": false,
-            "consentRequired": false,
-            "standardFlowEnabled": true,
-            "implicitFlowEnabled": false,
-            "directAccessGrantsEnabled": false,
-            "serviceAccountsEnabled": false,
-            "publicClient": true,
-            "frontchannelLogout": false,
-            "protocol": "openid-connect",
-            "attributes": {
-                "realm_client": "false",
-                "post.logout.redirect.uris": "+",
-                "pkce.code.challenge.method": "S256"
-            },
-            "authenticationFlowBindingOverrides": {},
-            "fullScopeAllowed": false,
-            "nodeReRegistrationTimeout": 0,
-            "protocolMappers": [
-                {
-                "id": "9a68ec2d-943d-49cb-9fdd-cd821d606210",
-                "name": "audience resolve",
-                "protocol": "openid-connect",
-                "protocolMapper": "oidc-audience-resolve-mapper",
-                "consentRequired": false,
-                "config": {}
-                }
-            ],
-            "defaultClientScopes": [
-                "service_account",
-                "web-origins",
-                "acr",
-                "address",
-                "administrator-scope",
-                "phone",
-                "profile",
-                "roles",
-                "microprofile-jwt",
-                "basic",
-                "email"
-            ],
-            "optionalClientScopes": []
-            },
             {
                 "id": "9a6a2560-c6be-4493-8bd5-3fdc4522d82b",
                 "clientId": {{ .Values.apps.accounts.client.id | quote }},

applications/jupyterlab/build_local.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -ex
+
+# Set the platform flag if we're on ARM
+arch=$(uname -m)
+if [[ "$arch" == "arm64" || "$arch" == "aarch64" ]]; then
+    platform_flag="--platform linux/amd64"
+else
+    platform_flag=""
+fi
+
+time DOCKER_BUILDKIT=1 docker build $platform_flag -t myjlab -f Dockerfile .
+

applications/jupyterlab/pip_omv_info.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+# A script to print info versions of packages in the JupyterLab container
+
+docker run -t --rm --entrypoint /bin/bash myjlab  -c "pip3 list"
+echo "--------------------------------------------------------"
+docker run -t --rm --entrypoint /bin/bash myjlab  -c "omv list -V"

applications/jupyterlab/rebuild_local.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -ex
+
+# Set the platform flag if we're on ARM
+arch=$(uname -m)
+if [[ "$arch" == "arm64" || "$arch" == "aarch64" ]]; then
+    platform_flag="--platform linux/amd64"
+else
+    platform_flag=""
+fi
+
+time DOCKER_BUILDKIT=1 docker build $platform_flag -t myjlab -f Dockerfile  --no-cache .
+

applications/jupyterlab/requirements.txt

@@ -6,7 +6,7 @@ neurotune
 neuron
 
 # Install specific version of NetPyNE
-git+https://github.com/Neurosim-lab/netpyne.git@osbv2#egg=netpyne
+git+https://github.com/Neurosim-lab/netpyne.git@osbv2-dev#egg=netpyne
 
 ####  Other simulators
 

applications/jupyterlab/run_local.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+# A script to run the JupyterLab container locally (build it first with ./build_local.sh)
+
+docker run --network host -it --rm  --name myjupyterlab myjlab 
+
+

applications/netpyne/Dockerfile

@@ -16,7 +16,7 @@ RUN yarn build-dev
 ### Download on a separate stage to run in parallel with buildkit
 FROM quay.io/jupyter/base-notebook:python-3.12 AS downloads
 USER root
-RUN wget --no-check-certificate -O /nyhead.mat https://www.parralab.org/nyhead/sa_nyhead.mat
+RUN apt-get update -qq && apt-get install aria2 -y && aria2c -j10 -x 10 https://www.parralab.org/nyhead/sa_nyhead.mat && mv sa_nyhead.mat /nyhead.mat
 
 ###
 FROM quay.io/jupyter/base-notebook:python-3.12
@@ -27,10 +27,13 @@ ENV NP_LFPYKIT_HEAD_FILE=/home/jovyan/nyhead.mat
 
 USER root
 
+### Some helpful aliases
+RUN echo -e '\n\nalias cd..="cd .." \nalias h=history \nalias ll="ls -alt" \n' >> ~/.bashrc
+
 RUN rm -rf /var/lib/apt/lists
 RUN apt-get update -qq &&\
     apt-get install python3-tk vim nano unzip git make libtool g++ -qq pkg-config libfreetype6-dev libpng-dev libopenmpi-dev -y
-RUN apt-get install openjdk-11-jre-headless -y
+RUN apt-get install openjdk-11-jre-headless htop ncdu -y
 # RUN conda install python=3.7 -y
 
 
@@ -87,7 +90,7 @@ RUN ln -s /opt/workspace workspace
 
 # RUN jupyter labextension disable @jupyterlab/hub-extension
 
-COPY --from=downloads --chown=1000:1000 /nyhead.mat $NP_LFPYKIT_HEAD_FILE
+COPY --from=downloads --chown=1000:1000 nyhead.mat $NP_LFPYKIT_HEAD_FILE
 COPY --from=jsbuild --chown=1000:1000 $FOLDER/webapp/build webapp/build
 
 RUN chown -R $NB_UID /home/jovyan/.jupyter

applications/netpyne/deploy/values.yaml

@@ -1,6 +1,6 @@
 harness:
   subdomain: netpyne
-  secured: true
+  secured: false
   uri_role_mapping: []
   service:
     auto: false
@@ -21,8 +21,8 @@ harness:
     hard:
     - jupyterhub
     git:
-      - url: https://github.com/MetaCell/NetPyNE-UI.git
-        branch_tag: 1.1.0
+      - url: https://github.com/OpenSourceBrain/NetPyNE-UI.git
+        branch_tag: osbv2_tests
 singleuser:
   cpu:
     limit: 1

applications/netpyne/overrides/requirements.txt

@@ -71,7 +71,7 @@ jupyterthemes==0.20.0
 kiwisolver==1.4.8
 lesscpy==0.15.1
 LFPykit==0.5.1
-libNeuroML==0.5.1
+libNeuroML==0.6.7
 lxml==5.3.1
 MarkupSafe==1.1.1
 matplotlib==3.10.0
@@ -88,9 +88,9 @@ nbconvert==5.6.1
 nbformat==5.2.0
 ndindex==1.9.2
 nest-asyncio==1.6.0
-netpyne @ git+https://github.com/Neurosim-lab/netpyne.git@3d633bcda9a3ab3fe4a90b7c705cd3692a729185
+netpyne @ git+https://github.com/Neurosim-lab/netpyne.git@osbv2-dev#egg=netpyne
 networkx==3.4.2
-neuromllite==0.5.4
+neuromllite==0.6.1
 NEURON==8.2.6
 notebook==6.4.5
 notebook_shim==0.2.3
@@ -117,9 +117,9 @@ pycparser==2.22
 pyecore==0.15.2
 pygeppetto==0.9.0
 Pygments==2.19.1
-PyLEMS==0.5.9
+PyLEMS==0.6.8
 pymongo==4.11.1
-pyNeuroML==1.0.10
+pyNeuroML==1.3.21
 pyparsing==3.2.1
 pytest==6.2.5
 python-dateutil==2.9.0.post0

applications/netpyne/run_local.sh

@@ -3,6 +3,6 @@ set -e
 
 # A script to run the NetPyNE container locally (build it first with ./build_local.sh)
 
-docker run --network host  -it --rm  --name mynp mynetpyneosb
+docker run --network host -v $PWD/shared:/opt/workspace/local:rw -it --rm  --name mynp mynetpyneosb
 
 

applications/osb-portal/src/components/MainDrawer/MainDrawer.tsx

@@ -338,7 +338,7 @@ export const MainDrawer = (props: {
                 <ListItemIcon>
                   <HomeIcon />
                 </ListItemIcon>
-                <ListItemText primary="Main site" />
+                <ListItemText primary="OSB homepage" />
               </ListItemButton>
 
               <ListItemButton 

applications/osb-portal/src/components/dialogs/AboutDialog.tsx

@@ -18,8 +18,8 @@ const styles = {
 
 export const AboutContent = (props: any) => {
 
-  const logoMetaCell = "/images/metacell.png";
-  const logoWellcome = "/images/wellcome.png";
+  const logoMetaCell = "https://raw.githubusercontent.com/OpenSourceBrain/OSBv2/refs/heads/master/applications/osb-portal/src/assets/images/metacell.png";
+  const logoWellcome = "https://raw.githubusercontent.com/OpenSourceBrain/OSBv2/refs/heads/master/applications/osb-portal/src/assets/images/wellcome.png";
 
   return (
     <Box sx={styles.paper}>
@@ -60,9 +60,16 @@ export const AboutContent = (props: any) => {
         <Typography variant="body1">
           OSBv2 is being developed by the{" "}
           <Link href="http://silverlab.org/" target="_blank" underline="hover">
-            Silver Lab at University College London
+            Silver Lab
           </Link>{" "}
-          in collaboration with{" "}
+          and the{" "}
+          <Link href="http://openneuroai.org/" target="_blank" underline="hover">
+            Gleeson Lab
+          </Link>{" "}
+           at {" "}
+          <Link href="http://ucl.ac.uk/" target="_blank" underline="hover">
+            University College London
+          </Link>, in collaboration with{" "}
           <Link href="https://metacell.us" target="_blank" underline="hover">
             MetaCell
           </Link>

applications/workspaces/api/openapi.yaml

@@ -271,6 +271,8 @@ paths:
             security:
                 -
                     bearerAuth: []
+                -
+                    cookieAuth: []
             operationId: workspaces.controllers.workspace_controller.setthumbnail
             summary: Sets the thumbnail of the workspace.
         parameters:
@@ -302,6 +304,8 @@ paths:
             security:
                 -
                     bearerAuth: []
+                -
+                    cookieAuth: []
             operationId: workspaces.controllers.workspace_controller.addimage
             summary: Adds and image to the workspace.
         parameters:
@@ -324,6 +328,8 @@ paths:
             security:
                 -
                     bearerAuth: []
+                -
+                    cookieAuth: []
             operationId: delimage
             summary: Delete a Workspace Image from the workspace.
             x-openapi-router-controller: workspaces.controllers.workspace_controller
@@ -365,6 +371,8 @@ paths:
             security:
                 -
                     bearerAuth: []
+                -
+                    cookieAuth: []
             summary: Used to save a WorkspaceResource to the repository.
     '/workspaceresource/{id}':
         get:
@@ -382,6 +390,8 @@ paths:
             security:
                 -
                     bearerAuth: []
+                -
+                    cookieAuth: []
             summary: Used to retrieve a WorkspaceResource.
         put:
             requestBody:
@@ -1517,6 +1527,11 @@ components:
             bearerFormat: JWT
             type: http
             x-bearerInfoFunc: cloudharness.auth.AuthClient.decode_token
+        cookieAuth:
+            type: apiKey
+            name: kc-access
+            in: cookie
+            x-apikeyInfoFunc: cloudharness.auth.decode_token
 tags:
     -
         name: rest

deployment/codefresh-dev.yaml

@@ -30,8 +30,8 @@ steps:
       clone_NetPyNE-UI_git_netpyne:
         title: Cloning NetPyNE-UI.git repository...
         type: git-clone
-        repo: https://github.com/MetaCell/NetPyNE-UI.git
-        revision: dev_netpyne_updates
+        repo: https://github.com/OpenSourceBrain/NetPyNE-UI.git
+        revision: osbv2_tests
         working_directory: applications/netpyne/dependencies/
         git: github
       clone_nwb-explorer_git_nwb-explorer:

local-scripts/osbv2-local.sh

@@ -153,8 +153,8 @@ deploy () {
         harness_deployment
 
         echo "-> running skaffold"
-        $SKAFFOLD dev --cleanup=false || { notify_fail "Failed: skaffold" ; minikube stop; }
-        #$SKAFFOLD dev || notify_fail "Failed: skaffold"
+        $SKAFFOLD dev --cleanup=false || { notify_fail "Failed: skaffold" ; }
+        minkube status
     popd
 }
 
@@ -193,10 +193,18 @@ harness_deployment() {
         if [ "YES" == "$LIVE" ]
         then
             echo "-> harnessing live configuration deployment, and deploying"
+            set -x
+            set -v
             harness-deployment ../cloud-harness . -l -n ${OSB_NAMESPACE} -d osb.local -r gcr.io/metacellllc -e "local" -t "$LIVE_TAG" || notify_fail "Failed: harness-deployment (live)"
+            set +x
+            set +v
         else
             echo "-> harnessing development deployment"
+            set -x
+            set -v
             harness-deployment ../cloud-harness . -l  -n ${OSB_NAMESPACE} -d osb.local -dtls -e "local" ${DEPLOYMENT_APP:+-i $DEPLOYMENT_APP} || notify_fail "Failed: harness-deployment (dev)"
+            set +x
+            set +v
         fi
     popd
 }
@@ -242,10 +250,12 @@ deactivate_venv() {
 print_versions() {
     echo "** docker **"
     docker version
+    echo -e "\n** kubernetes **"
+    kubectl version
     echo -e "\n** minikube **"
     minikube version
     echo -e "\n** cloud harness **"
-    pushd "${CLOUD_HARNESS_DIR}" && git log --oneline | head -1 && popd
+    pushd "${CLOUD_HARNESS_DIR}" 2>&1 > /dev/null && git log --oneline | head -1 && popd 2>&1 > /dev/null
     echo -e "\n** helm **"
     helm version
     echo -e "\n** skaffold **"