Merge pull request #87 from communitiesuk/fmd-218-paketo-build

FMD-218: paketo build

.github/workflows/deploy.yml

@@ -17,20 +17,37 @@ on:
           - production
 
 jobs:
+  tag_version:
+    runs-on: ubuntu-latest
+    outputs:
+      version_to_tag: ${{ steps.tagging.outputs.tag_value }}
+    steps:
+      - id: tagging
+        run: |
+          echo "tag_value=$(echo '${{ github.ref }}' | sed -e 's,.*/\(.*\),\1,')" >> $GITHUB_OUTPUT
+  paketo_build:
+    permissions:
+      packages: write
+    needs: [ tag_version ]
+    uses: communitiesuk/funding-service-design-workflows/.github/workflows/package.yml@main
+    with:
+      assets_required: true
+      version_to_build: ${{ needs.tag_version.outputs.version_to_tag }}
+      owner: ${{ github.repository_owner }}
+      application: ${{ github.event.repository.name }}
   deployment:
     permissions:
       id-token: write # This is required for requesting the JWT
       contents: read  # This is required for actions/checkout
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment || 'test' }}
+    needs: [ tag_version, paketo_build ]
+    env:
+      IMAGE_LOCATION: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}:${{ github.ref_name == 'main' && 'latest' || needs.tag_version.outputs.version_to_tag }}
     steps:
       - name: Git clone the repository
         uses: actions/checkout@v3
 
-      - name: Build static assets
-        run: |
-           ./build.sh
-
       - name: Get current date
         id: currentdatetime
         run: echo "::set-output name=datetime::$(date +'%Y%m%d%H%M%S')"
@@ -46,9 +63,10 @@ jobs:
         run: |
           curl -Lo aws-copilot https://github.com/aws/copilot-cli/releases/latest/download/copilot-linux && chmod +x aws-copilot && sudo mv aws-copilot /usr/local/bin/copilot
 
-      - name: Inject git sha
+      - name: Inject env specific values into manifest
         run: |
           yq -i '.variables.GITHUB_SHA = "${{ github.sha }}"' copilot/data-frontend/manifest.yml
+          yq -i '.image.location = "${{ env.IMAGE_LOCATION }}"' copilot/data-frontend/manifest.yml
 
       - name: Copilot deploy
         run: |

.github/workflows/test.yml

@@ -11,10 +11,6 @@ jobs:
         cache: pip   # caching pip dependencies
     - name: Install dependencies
       run: pip install -r requirements-dev.txt
-    - name: build static assets
-      env:
-        FLASK_ENV: development
-      run: ./build.sh
     - name: Run tests
       env:
         FLASK_ENV: development

.gitignore

@@ -128,8 +128,8 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-app/static
-govuk_components
+app/static/dist
+.webassets-manifest
 
 # ide
 .idea/

Procfile

@@ -0,0 +1 @@
+web: gunicorn wsgi:app -c run/gunicorn/run.py --workers=3 --timeout 180

README.md

@@ -72,10 +72,9 @@ Then run:
 ### Get GOV.UK Frontend assets
 
 For convenience a shell script has been provided to download and extract the GOV.UK Frontend distribution assets
-Windows users will need to add a compatability layer such as [WSL](https://learn.microsoft.com/en-us/windows/wsl/install) or Git Bash to run the script
 
-```
-./build.sh
+```shell
+python build.py
 ```
 
 ### Setup pre-commit checks
@@ -84,6 +83,9 @@ Windows users will need to add a compatability layer such as [WSL](https://learn
 * Pre-commit hooks can either be installed using pip `pip install pre-commit` or homebrew (for Mac users)`brew install pre-commit`
 * From your checkout directory run `pre-commit install` to set up the git hook scripts
 
+### Docker Compose (recommended)
+To run the app alongside other related microservices see https://github.com/communitiesuk/funding-service-design-post-award-docker-runner
+
 ### Run app
 To run the front-end app locally, you can run the following:
 ```
@@ -103,3 +105,8 @@ To run the tests:
 ```shell
 python -m pytest --cov=app --cov-report=term-missing --cov-branch
 ```
+
+## Deployment
+`main` branch is continuously deployed to the AWS Test environment, deployments to the Dev and Production environments are triggered by a [manual Github Actions workflow](https://github.com/communitiesuk/funding-service-design-post-award-data-frontend/actions/workflows/deploy.yml).
+
+On the deployed environments we use [Paketo buildpacks](https://paketo.io) rather than the local Dockerfile.

app/__init__.py

@@ -1,5 +1,5 @@
 from flask import Flask
-from flask_assets import Bundle, Environment
+from flask_assets import Environment
 from flask_talisman import Talisman
 from flask_wtf.csrf import CSRFProtect
 from fsd_utils import init_sentry
@@ -9,6 +9,7 @@
 from govuk_frontend_wtf.main import WTFormsHelpers
 from jinja2 import ChoiceLoader, PackageLoader, PrefixLoader
 
+import static_assets
 from config import Config
 
 assets = Environment()
@@ -47,21 +48,15 @@ def create_app(config_class=Config):
     }
 
     # Initialise app extensions
+    app.static_folder = "static/dist/"
     assets.init_app(app)
-    talisman.init_app(app, content_security_policy=csp, force_https=False)
-    WTFormsHelpers(app)
 
-    # Create static asset bundles
-    css = Bundle(
-        "src/css/*.css", filters="cssmin", output="dist/css/custom-%(version)s.min.css"
-    )
-    js = Bundle(
-        "src/js/*.js", filters="jsmin", output="dist/js/custom-%(version)s.min.js"
+    static_assets.init_assets(
+        app, auto_build=Config.AUTO_BUILD_ASSETS, static_folder="static/dist"
     )
-    if "css" not in assets:
-        assets.register("css", css)
-    if "js" not in assets:
-        assets.register("js", js)
+
+    talisman.init_app(app, content_security_policy=csp, force_https=False)
+    WTFormsHelpers(app)
 
     # Register blueprints
     from app.main import bp as main_bp

app/static/src/css/custom.css

@@ -1,5 +1,5 @@
 .govuk-footer__copyright-logo {
-    background-image: url("/static/images/govuk-crest.png")
+    background-image: url("/static/govuk-frontend/images/govuk-crest.png")
 }
 
 

app/templates/base.html

@@ -14,7 +14,7 @@
   <meta name="description" content="{{config['SERVICE_NAME']}}">
   <meta name="keywords" content="GOV.UK, govuk, gov, government, uk, frontend, ui, user interface, jinja, python, flask, port, template, templating, macro, component, design system, html, forms, wtf, wtforms, widget, widgets, demo, example">
   <meta name="author" content="{{config['DEPARTMENT_NAME']}}">
-  <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='govuk-frontend-4.6.0.min.css') }}" />
+  <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='govuk-frontend/govuk-frontend-4.6.0.min.css') }}" />
   {% assets "css" %}<link href="{{ ASSET_URL }}" rel="stylesheet">{% endassets %}
 {% endblock %}
 
@@ -93,7 +93,7 @@ <h2 class="govuk-visually-hidden">Support links</h2>
 
 {% block bodyEnd %}
   <!--[if gt IE 8]><!-->
-  <script src="{{ url_for('static', filename='govuk-frontend-4.6.0.min.js') }}"> </script>
+  <script src="{{ url_for('static', filename='govuk-frontend/govuk-frontend-4.6.0.min.js') }}"> </script>
   <script>window.GOVUKFrontend.initAll()</script>
   <!--<![endif]-->
   {% assets "js" %}<script type="text/javascript" src="{{ ASSET_URL }}"></script>{% endassets %}

app/templates/main/login.html

@@ -6,7 +6,7 @@
   <meta name="description" content="{{config['SERVICE_NAME']}}">
   <meta name="keywords" content="GOV.UK, govuk, gov, government, uk, frontend, ui, user interface, jinja, python, flask, port, template, templating, macro, component, design system, html, forms, wtf, wtforms, widget, widgets, demo, example">
   <meta name="author" content="{{config['DEPARTMENT_NAME']}}">
-  <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='govuk-frontend-4.6.0.min.css') }}" />
+  <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='govuk-frontend/govuk-frontend-4.6.0.min.css') }}" />
   {% assets "css" %}<link href="{{ ASSET_URL }}" rel="stylesheet">{% endassets %}
 {% endblock %}
 
@@ -47,7 +47,7 @@ <h2 class="govuk-heading-l govuk-!-margin-top-4">Get help</h2>
 {{ govukFooter({}) }}
 
 {% block bodyEnd %}
-  <script src="{{ url_for('static', filename='govuk-frontend-4.6.0.min.js') }}"> </script>
+  <script src="{{ url_for('static', filename='govuk-frontend/govuk-frontend-4.6.0.min.js') }}"> </script>
   <script>window.GOVUKFrontend.initAll()</script>
   {% assets "js" %}<script type="text/javascript" src="{{ ASSET_URL }}"></script>{% endassets %}
 {% endblock %}

build.py

@@ -0,0 +1,86 @@
+import glob
+import os
+import shutil
+import urllib.request
+import zipfile
+
+import static_assets
+
+
+def build_govuk_assets(static_dist_root="app/static/dist"):
+    GOVUK_FRONTEND_VERSION = "4.6.0"
+    DIST_ROOT = "./" + static_dist_root
+    GOVUK_DIR = "/govuk-frontend"
+    GOVUK_URL = (
+        "https://github.com/alphagov/govuk-frontend/releases/download/"
+        f"v{GOVUK_FRONTEND_VERSION}/release-v{GOVUK_FRONTEND_VERSION}.zip"
+    )
+    ZIP_FILE = "./govuk_frontend.zip"
+    DIST_PATH = DIST_ROOT + GOVUK_DIR
+    ASSETS_DIR = "/assets"
+    ASSETS_PATH = DIST_PATH + ASSETS_DIR
+
+    # Checks if GovUK Frontend Assets already built
+    print(DIST_PATH)
+    if os.path.exists(DIST_PATH):
+        print(
+            "GovUK Frontend assets already built. If you require a rebuild manually run build.build_govuk_assets"
+        )
+        return True
+
+    # Download zips from GOVUK_URL
+    # There is a known problem on Mac where one must manually
+    # run the script "Install Certificates.command" found
+    # in the python application folder for this to work.
+
+    print("Downloading static file zip.")
+    urllib.request.urlretrieve(GOVUK_URL, ZIP_FILE)  # nosec
+
+    # Attempts to delete the old files, states if
+    # one doesn't exist.
+
+    print("Deleting old " + DIST_PATH)
+    try:
+        shutil.rmtree(DIST_PATH)
+    except FileNotFoundError:
+        print("No old " + DIST_PATH + " to remove.")
+
+    # Extract the previously downloaded zip to DIST_PATH
+
+    print("Unzipping file to " + DIST_PATH + "...")
+    with zipfile.ZipFile(ZIP_FILE, "r") as zip_ref:
+        zip_ref.extractall(DIST_PATH)
+
+    # Move files from ASSETS_PATH to DIST_PATH
+
+    print("Moving files from " + ASSETS_PATH + " to " + DIST_PATH)
+    for file_to_move in os.listdir(ASSETS_PATH):
+        shutil.move("/".join([ASSETS_PATH, file_to_move]), DIST_PATH)
+
+    # Update relative paths
+
+    print("Updating relative paths in css files to " + GOVUK_DIR)
+    cwd = os.getcwd()
+    os.chdir(DIST_PATH)
+    for css_file in glob.glob("*.css"):
+        # Read in the file
+        with open(css_file, "r") as file:
+            filedata = file.read()
+
+        # Replace the target string
+        filedata = filedata.replace(ASSETS_DIR, "/static/" + GOVUK_DIR)
+
+        # Write the file out again
+        with open(css_file, "w") as file:
+            file.write(filedata)
+    os.chdir(cwd)
+
+    # Delete temp files
+    print("Deleting " + ASSETS_PATH)
+    shutil.rmtree(ASSETS_PATH)
+    os.remove(ZIP_FILE)
+
+
+if __name__ == "__main__":
+    build_govuk_assets()
+    static_assets.build_bundles(static_folder="app/static/dist")

config/envs/default.py

@@ -36,3 +36,5 @@ class DefaultConfig(object):
     RSA256_PUBLIC_KEY_BASE64 = os.getenv("RSA256_PUBLIC_KEY_BASE64")
     if RSA256_PUBLIC_KEY_BASE64:
         RSA256_PUBLIC_KEY = base64.b64decode(RSA256_PUBLIC_KEY_BASE64).decode()
+
+    AUTO_BUILD_ASSETS = False

config/envs/development.py

@@ -23,3 +23,4 @@ class DevelopmentConfig(DefaultConfig):
         "roles": [],
         "highest_role_map": {},
     }
+    AUTO_BUILD_ASSETS = True

config/envs/unit_test.py

@@ -15,3 +15,5 @@ class UnitTestConfig(DefaultConfig):
         )
         with open(_test_public_key_path, mode="rb") as public_key_file:
             RSA256_PUBLIC_KEY = public_key_file.read()
+
+        AUTO_BUILD_ASSETS = True

copilot/data-frontend/manifest.yml

@@ -17,8 +17,7 @@ http:
 
 # Configuration for your containers and service.
 image:
-  # Docker build arguments. For additional overrides: https://aws.github.io/copilot-cli/docs/manifest/lb-web-service/#image-build
-  build: Dockerfile
+  # Image location will be injected here by Github Actions
   # Port exposed through your container to route traffic to it.
   port: 8080
 

project.toml

@@ -0,0 +1,6 @@
+[ _ ]
+schema-version = "0.2"
+
+[[ io.buildpacks.build.env ]]
+  name = "BP_CPYTHON_VERSION"
+  value = "3.11.*" # any valid semver constraints (e.g. 3.6.7, 3.*) are acceptable

static_assets.py

@@ -0,0 +1,44 @@
+"""Compile static assets."""
+
+from os import path
+
+from flask import Flask
+from flask_assets import Bundle, Environment
+
+
+def init_assets(app=None, auto_build=False, static_folder="app/static/dist"):
+    app = app or Flask(__name__)
+    app.static_folder = static_folder
+    with app.app_context():
+        env = Environment(app)
+        env.load_path = [path.join(path.dirname(__file__), "app/static/src")]
+        # Paketo doesn't support automatic rebuilding.
+        env.auto_build = auto_build
+        # This file needs to be shipped with your code.
+        env.manifest = "file"
+
+        js = Bundle("./js/*.js", filters="jsmin", output="js/custom-%(version)s.min.js")
+
+        css = Bundle(
+            Bundle(
+                "./css/*.css",
+                filters="cssmin",
+                output="css/custom-%(version)s.min.css",
+            )
+        )
+
+        env.register("js", js)
+        env.register("css", css)
+
+        bundles = [css, js]
+        return bundles
+
+
+def build_bundles(static_folder="static/dist"):
+    bundles = init_assets(static_folder=static_folder)
+    for bundle in bundles:
+        bundle.build()
+
+
+if __name__ == "__main__":
+    build_bundles()