Feat: improve proof-of-concept app (#116)

Author: lalver1

.devcontainer/Dockerfile

@@ -11,23 +11,10 @@ RUN pip install --no-cache-dir -r docs/requirements.txt
 # install streamlit requirements
 RUN pip install --no-cache-dir -r streamlit_app/requirements.txt
 
-# install AWS CLI and Copilot CLI (requires root permissions)
+# install AWS Copilot CLI (requires root permissions)
 USER root
-# download AWS CLIs to /tmp to avoid write error (23) from curl command
+# download AWS Copilot CLI to /tmp to avoid write error (23) from curl command
 WORKDIR /tmp
-RUN HOST_ARCH=$(uname -m) && \
-    case "$HOST_ARCH" in \
-      x86_64) HOST_ARCH="amd64" ;; \
-      aarch64) HOST_ARCH="arm64" ;; \
-    esac \
-    && if [ "$HOST_ARCH" = "amd64" ]; then \
-    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"; \
-    elif [ "$HOST_ARCH" = "arm64" ]; then \
-    curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip"; \
-    fi \
-    && unzip awscliv2.zip \
-    && ./aws/install \
-    && rm -rf aws awscliv2.zip
 
 RUN curl -Lo copilot https://github.com/aws/copilot-cli/releases/latest/download/copilot-linux \
     && chmod +x copilot \

appcontainer/Dockerfile

@@ -52,6 +52,23 @@ RUN useradd --create-home --shell /bin/bash $USER && \
     apt-get install -qq --no-install-recommends build-essential nginx gettext && \
     python -m pip install --upgrade pip
 
+# install AWS CLI (requires root permissions)
+# download AWS CLI to /tmp to avoid write error (23) from curl command
+WORKDIR /tmp
+RUN HOST_ARCH=$(uname -m) && \
+    case "$HOST_ARCH" in \
+      x86_64) HOST_ARCH="amd64" ;; \
+      aarch64) HOST_ARCH="arm64" ;; \
+    esac \
+    && if [ "$HOST_ARCH" = "amd64" ]; then \
+    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"; \
+    elif [ "$HOST_ARCH" = "arm64" ]; then \
+    curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip"; \
+    fi \
+    && unzip awscliv2.zip \
+    && ./aws/install \
+    && rm -rf aws awscliv2.zip
+
 # enter source directory
 WORKDIR /$USER
 

bin/start_aws.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -eu
+
+#
+# S3 bucket name is injected by Copilot as an environment variable
+# since it was created via copilot storage init --name pems-db, the variable is 'PEMSDB_NAME'
+S3_BUCKET_NAME="$PEMSDB_NAME"
+S3_FIXTURE_PATH="fixtures.json"
+LOCAL_FIXTURE_PATH="fixtures.json"
+
+echo "Downloading $S3_FIXTURE_PATH from bucket $S3_BUCKET_NAME"
+aws s3 cp "s3://${S3_BUCKET_NAME}/${S3_FIXTURE_PATH}" "${LOCAL_FIXTURE_PATH}"
+echo "Download complete"
+
+# initialize Django
+
+bin/init.sh
+
+# effectively reset database by loading downloaded fixtures into the database
+echo "Loading data from ${LOCAL_FIXTURE_PATH}"
+python manage.py loaddata "${LOCAL_FIXTURE_PATH}"
+echo "Data loading complete"
+
+# start the web server
+
+nginx
+
+# start the application server
+
+python -m gunicorn -c $GUNICORN_CONF pems.wsgi

copilot/.workspace renamed to infra/copilot/.workspace

@@ -4,16 +4,21 @@
 
 # Your service name will be used in naming your resources like log groups, ECS services, etc.
 name: streamlit
-type: Backend Service
+type: Load Balanced Web Service
 
-# Your service is reachable at "http://streamlit.${COPILOT_SERVICE_DISCOVERY_ENDPOINT}:8501" but is not public.
+# Distribute traffic to your service.
+http:
+  # Requests to this path will be forwarded to your service.
+  # To match all requests you can use the "/" path.
+  path: "/streamlit"
+  healthcheck: "/streamlit/_stcore/health"
 
 # Configuration for your containers and service.
 image:
   # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/backend-service/#image-build
   build:
-    dockerfile: streamlit_app/Dockerfile
-    context: .
+    dockerfile: ../streamlit_app/Dockerfile
+    context: ../
   # Port exposed through your container to route traffic to it.
   port: 8501
 
@@ -30,9 +35,8 @@ network:
 
 # Optional fields for more advanced use-cases.
 #
-#variables:                    # Pass environment variables as key value pairs.
-#  LOG_LEVEL: info
-
+variables: # Pass environment variables as key value pairs.
+  STREAMLIT_BASE_URL: /streamlit
 #secrets:                      # Pass secrets from AWS Systems Manager (SSM) Parameter Store.
 #  GITHUB_TOKEN: GITHUB_TOKEN  # The key is the name of the environment variable, the value is the name of the SSM parameter.
 

copilot/environments/dev/manifest.yml renamed to infra/copilot/environments/dev/manifest.yml

@@ -0,0 +1,93 @@
+Parameters:
+  App:
+    Type: String
+    Description: Your application's name.
+  Env:
+    Type: String
+    Description: The environment name your service, job, or workflow is being deployed to.
+  Name:
+    Type: String
+    Description: Your workload's name.
+Resources:
+  pemsdbBucket:
+    Metadata:
+      "aws:copilot:description": "An Amazon S3 bucket to store and retrieve objects for pems-db"
+    Type: AWS::S3::Bucket
+    Properties:
+      VersioningConfiguration:
+        Status: Enabled
+      AccessControl: Private
+      BucketEncryption:
+        ServerSideEncryptionConfiguration:
+          - ServerSideEncryptionByDefault:
+              SSEAlgorithm: AES256
+      PublicAccessBlockConfiguration:
+        BlockPublicAcls: true
+        BlockPublicPolicy: true
+        IgnorePublicAcls: true
+        RestrictPublicBuckets: true
+      OwnershipControls:
+        Rules:
+          - ObjectOwnership: BucketOwnerEnforced
+      LifecycleConfiguration:
+        Rules:
+          - Id: ExpireNonCurrentObjects
+            Status: Enabled
+            NoncurrentVersionExpirationInDays: 30
+            AbortIncompleteMultipartUpload:
+              DaysAfterInitiation: 1
+
+  pemsdbBucketPolicy:
+    Metadata:
+      "aws:copilot:description": "A bucket policy to deny unencrypted access to the bucket and its contents"
+    Type: AWS::S3::BucketPolicy
+    DeletionPolicy: Retain
+    Properties:
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Sid: ForceHTTPS
+            Effect: Deny
+            Principal: "*"
+            Action: "s3:*"
+            Resource:
+              - !Sub ${ pemsdbBucket.Arn}/*
+              - !Sub ${ pemsdbBucket.Arn}
+            Condition:
+              Bool:
+                "aws:SecureTransport": false
+      Bucket: !Ref pemsdbBucket
+
+  pemsdbAccessPolicy:
+    Metadata:
+      "aws:copilot:description": "An IAM ManagedPolicy for your service to access the pems-db bucket"
+    Type: AWS::IAM::ManagedPolicy
+    Properties:
+      Description: !Sub
+        - Grants CRUD access to the S3 bucket ${Bucket}
+        - { Bucket: !Ref pemsdbBucket }
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Sid: S3ObjectActions
+            Effect: Allow
+            Action:
+              - s3:GetObject
+              - s3:PutObject
+              - s3:PutObjectACL
+              - s3:PutObjectTagging
+              - s3:DeleteObject
+              - s3:RestoreObject
+            Resource: !Sub ${ pemsdbBucket.Arn}/*
+          - Sid: S3ListAction
+            Effect: Allow
+            Action: s3:ListBucket
+            Resource: !Sub ${ pemsdbBucket.Arn}
+
+Outputs:
+  pemsdbName:
+    Description: "The name of a user-defined bucket."
+    Value: !Ref pemsdbBucket
+  pemsdbAccessPolicy:
+    Description: "The IAM::ManagedPolicy to attach to the task role"
+    Value: !Ref pemsdbAccessPolicy

copilot/streamlit/manifest.yml renamed to infra/copilot/streamlit/manifest.yml

@@ -10,37 +10,38 @@ type: Load Balanced Web Service
 http:
   # Requests to this path will be forwarded to your service.
   # To match all requests you can use the "/" path.
-  path: '/'
+  path: "/"
   # You can specify a custom health check path. The default is "/".
-  healthcheck: '/healthcheck'
+  healthcheck: "/healthcheck"
 
 # Configuration for your containers and service.
 image:
   # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/lb-web-service/#image-build
   build:
-    dockerfile: appcontainer/Dockerfile
-    context: .
+    dockerfile: ../appcontainer/Dockerfile
+    context: ../
   # Port exposed through your container to route traffic to it.
   port: 8000
 
-cpu: 256       # Number of CPU units for the task.
-memory: 512    # Amount of memory in MiB used by the task.
-platform: linux/x86_64  # See https://aws.github.io/copilot-cli/docs/manifest/lb-web-service/#platform
-count: 1       # Number of tasks that should be running in your service.
-exec: true     # Enable running commands in your container.
+cpu: 256 # Number of CPU units for the task.
+memory: 512 # Amount of memory in MiB used by the task.
+platform: linux/x86_64 # See https://aws.github.io/copilot-cli/docs/manifest/lb-web-service/#platform
+count: 1 # Number of tasks that should be running in your service.
+exec: true # Enable running commands in your container.
+command: bin/start_aws.sh
 network:
   connect: true # Enable Service Connect for intra-environment traffic between services.
 
 # storage:
-  # readonly_fs: true       # Limit to read-only access to mounted root filesystems.
+# readonly_fs: true       # Limit to read-only access to mounted root filesystems.
 
 # Optional fields for more advanced use-cases.
 #
-#variables:                    # Pass environment variables as key value pairs.
-#  LOG_LEVEL: info
+variables: # Pass environment variables as key value pairs.
+  STREAMLIT_URL: /streamlit
 
-secrets:                      # Pass secrets from AWS Systems Manager (SSM) Parameter Store.
- DJANGO_ALLOWED_HOSTS: /pems/web/DJANGO_ALLOWED_HOSTS  # The key is the name of the environment variable, the value is the name of the SSM parameter.
+secrets: # Pass secrets from AWS Systems Manager (SSM) Parameter Store.
+  DJANGO_ALLOWED_HOSTS: /pems/web/DJANGO_ALLOWED_HOSTS # The key is the name of the environment variable, the value is the name of the SSM parameter.
 
 # You can override any of the values defined above by environment.
 #environments:

infra/copilot/web/addons/pems-db.yml

@@ -3,9 +3,16 @@
 """
 
 from pems import __version__
+from django.conf import settings
 
 
 def pems_version(request):
     """Context processor adds information about the PeMS application's version."""
 
     return {"pems_version": __version__}
+
+
+def streamlit(request):
+    """Context processor adds Streamlit-related information."""
+
+    return {"streamlit": {"url": settings.STREAMLIT_URL}}

copilot/web/manifest.yml renamed to infra/copilot/web/manifest.yml

@@ -9,16 +9,11 @@
         <div class="col-lg-4 border">
             <h2>Form</h2>
         </div>
-        <div class="col-lg-8 border">
-            <h2>Chart</h2>
-        </div>
     </div>
-    <div class="row">
-        <div class="col-lg-4 border">
-            <h2>Details for {{ current_district.name }}</h2>
-        </div>
-        <div class="col-lg-8 border">
-            <h2>Map</h2>
+    <div class="row" style="min-height: 450px;">
+        <div class="col-lg-12 border">
+            <iframe class="w-100 h-100" src="{{ streamlit.url }}/stations--stations?embed=true&district_number={{ current_district.number }}">
+            </iframe>
         </div>
     </div>
 {% endblock districts-content %}

pems/core/context_processors.py

@@ -13,7 +13,6 @@
                 {% endblock sidebar %}
             </div>
             <div class="col-lg-10 pb-lg-5">
-
                 {% block districts-content %}
                     <div class="row">
                         <div class="col-lg-4 border">
@@ -23,16 +22,13 @@ <h2>Form</h2>
                             <h2>Chart</h2>
                         </div>
                     </div>
-                    <div class="row">
-                        <div class="col-lg-4 border">
-                            <h2>Details</h2>
-                        </div>
-                        <div class="col-lg-8 border">
-                            <h2>Map</h2>
+                    <div class="row" style="min-height: 450px;">
+                        <div class="col-lg-12 border">
+                            <iframe class="w-100 h-100" src="{{ streamlit.url }}/stations--stations?embed=true">
+                            </iframe>
                         </div>
                     </div>
                 {% endblock districts-content %}
-
             </div>
         </div>
     </div>

pems/districts/templates/districts/district.html

@@ -33,7 +33,6 @@ def _filter_empty(ls):
     "django.contrib.staticfiles",
     "pems.core",
     "pems.districts",
-    "pems.streamlit_sample",
 ]
 
 MIDDLEWARE = [
@@ -61,6 +60,7 @@ def _filter_empty(ls):
                 "django.contrib.auth.context_processors.auth",
                 "django.contrib.messages.context_processors.messages",
                 "pems.core.context_processors.pems_version",
+                "pems.core.context_processors.streamlit",
             ],
         },
     },
@@ -138,3 +138,6 @@ def _filter_empty(ls):
 # https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field
 
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+
+# Streamlit settings
+STREAMLIT_URL = os.environ.get("STREAMLIT_URL", "http://localhost:8501")

pems/districts/templates/districts/index.html

@@ -5,5 +5,4 @@
     path("", include("pems.core.urls")),
     path("admin/", admin.site.urls),
     path("districts/", include("pems.districts.urls")),
-    path("streamlit/", include("pems.streamlit_sample.urls")),
 ]

pems/settings.py

@@ -33,8 +33,10 @@ ENV PYTHONPATH="$PYTHONPATH:/$USER/app"
 
 EXPOSE 8501
 
+COPY .streamlit .streamlit
+
 COPY streamlit_app streamlit_app
 
 RUN pip install -r streamlit_app/requirements.txt
 
-ENTRYPOINT ["streamlit", "run", "streamlit_app/main.py", "--server.port=8501", "--server.address=0.0.0.0"]
+ENTRYPOINT ["./streamlit_app/entrypoint.sh"]