E2e testing

.github/workflows/ci-build.yml

@@ -8,8 +8,8 @@ on:
       - develop
 
 jobs:
-  test:
-    name: Tests
+  lint-and-unit-test:
+    name: Lint & Unit Tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
@@ -20,10 +20,7 @@ jobs:
           node-version: 20
           cache: 'yarn'
       - name: Install dependencies
-        # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
-        run: |
-          sudo apt-get install libgconf-2-4
-          yarn --immutable
+        run: yarn --immutable
       - name: Run linting
         run: yarn lint:js
       - name: Run unit tests
@@ -34,6 +31,22 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true
+  cypress-tests:
+    name: Cypress Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+      - name: Setup Node.js
+        uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4
+        with:
+          node-version: 20
+          cache: 'yarn'
+      - name: Install dependencies
+        # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
+        run: |
+          sudo apt-get install libgconf-2-4
+          yarn --immutable
       - name: Run e2e tests
         run: yarn e2e
       - name: Upload Cypress screenshots
@@ -42,3 +55,174 @@ jobs:
         with:
           name: OperationsGateway Screenshots
           path: cypress/screenshots
+  playwright-tests-mocked:
+    name: Playwright Tests (mocked)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+      - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4
+        with:
+          node-version: 20
+          cache: 'yarn'
+
+      - name: Install dependencies
+        run: yarn --immutable
+
+      - name: Run playwright tests
+        run: yarn playwright:test:mocked
+
+      - name: Upload test report
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4
+        if: always()
+        with:
+          name: playwright-report-mocked-tests
+          path: playwright-report/
+          retention-days: 10
+  playwright-tests-real:
+    name: E2E Tests
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+
+      - name: Checkout OperationsGateway API
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+        with:
+          repository: ral-facilities/operationsgateway-api
+          ref: main
+          path: operationsgateway-api
+
+      # Install dependencies of python-ldap
+      - name: Install python-ldap dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libsasl2-dev python3.9-dev libldap2-dev libssl-dev
+
+      # Setup Python and environment dependencies (via cache)
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+      - name: Load Pip cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ubuntu-20.04-pip-3.9-${{ env.pythonLocation }}-${{ hashFiles('operationsgateway-api/.github/ci_requirements.txt') }}
+      - name: Install Poetry
+        run: pip install -r .github/ci_requirements.txt
+        working-directory: ./operationsgateway-api
+
+      # Install and start MongoDB
+      - name: Start MongoDB
+        uses: supercharge/[email protected]
+        with:
+          mongodb-version: '5.0'
+
+       # Configure correct paths in config files
+      - name: Configure private key path
+        run: yq -i ".auth.private_key_path = \"$GITHUB_WORKSPACE/id_rsa\"" .github/ci_config.yml
+        working-directory: ./operationsgateway-api
+      - name: Configure public key path
+        run: yq -i ".auth.public_key_path = \"$GITHUB_WORKSPACE/id_rsa.pub\"" .github/ci_config.yml
+        working-directory: ./operationsgateway-api
+      - name: Configure api host
+        run: yq -i ".app.host = \"0.0.0.0\"" .github/ci_config.yml
+        working-directory: ./operationsgateway-api
+      - name: Configure log config path
+        run: yq -i ".api.log_config_path = \"$GITHUB_WORKSPACE/operationsgateway-api/operationsgateway_api/logging.ini\"" .github/ci_ingest_echo_config.yml
+        working-directory: ./operationsgateway-api
+
+      # Read the database name from the config file and store it in an environment variable
+      - name: Get database name from ci_config.yml
+        run: echo "DATABASE_NAME=$(grep database_name .github/ci_config.yml | cut -d ':' -f 2 | tr -d '[:space:]')" >> $GITHUB_ENV
+        working-directory: ./operationsgateway-api
+
+      # Load Poetry virtual environment dependencies and install API dependencies
+      - name: Load Poetry cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: ubuntu-20.04-poetry-3.9-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
+      - name: Install dependencies
+        run: poetry install --without simulated-data
+        working-directory: ./operationsgateway-api
+
+      # Setup minio and create bucket
+      - name: Setup minio
+        run: |
+          docker run -d -p 9000:9000 --name minio \
+                     -e "MINIO_ACCESS_KEY=minioadmin" \
+                     -e "MINIO_SECRET_KEY=minioadmin" \
+                     -v /tmp/data:/data \
+                     -v /tmp/config:/root/.minio \
+                     minio/minio server /data
+          export AWS_ACCESS_KEY_ID=minioadmin
+          export AWS_SECRET_ACCESS_KEY=minioadmin
+          export AWS_EC2_METADATA_DISABLED=true
+          aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://og-actions-test
+
+      - name: Move CI config.yml to correct place
+        run: cp .github/ci_config.yml operationsgateway_api/config.yml
+        working-directory: ./operationsgateway-api
+      - name: Setup logging configuration
+        run: cp operationsgateway_api/logging.ini.example operationsgateway_api/logging.ini
+        working-directory: ./operationsgateway-api
+      - name: Create log file
+        run: touch "./logs.log"
+        working-directory: ./operationsgateway-api
+
+      - name: Create SSH private key file for auth
+        run: 'echo "$SSH_KEY_PRIVATE" > $GITHUB_WORKSPACE/id_rsa'
+        shell: bash
+        env:
+          SSH_KEY_PRIVATE: ${{secrets.SSH_PRIVATE_KEY_FOR_AUTH_OPENSSH}}
+
+      - name: Set permissions on private key file
+        run: chmod 600 $GITHUB_WORKSPACE/id_rsa
+
+      - name: Create SSH public key file for auth
+        run: 'echo "$SSH_KEY_PUBLIC" > $GITHUB_WORKSPACE/id_rsa.pub'
+        shell: bash
+        env:
+          SSH_KEY_PUBLIC: ${{secrets.SSH_PUBLIC_KEY_FOR_AUTH_OPENSSH}}
+
+      # Setup steps for Echo ingestion script
+      - name: Configure echo access key
+        run: yq -i ".echo.access_key = \"$ECHO_S3_ACCESS_KEY\"" .github/ci_ingest_echo_config.yml
+        env:
+          ECHO_S3_ACCESS_KEY: ${{secrets.ECHO_S3_ACCESS_KEY}}
+        working-directory: ./operationsgateway-api
+      - name: Configure echo secret key
+        run: yq -i ".echo.secret_key = \"$ECHO_S3_SECRET_KEY\"" .github/ci_ingest_echo_config.yml
+        env:
+          ECHO_S3_SECRET_KEY: ${{secrets.ECHO_S3_SECRET_KEY}}
+        working-directory: ./operationsgateway-api
+      - name: Copy config for Echo Ingest script to correct place
+        run: cp .github/ci_ingest_echo_config.yml util/realistic_data/config.yml
+        working-directory: ./operationsgateway-api
+
+      - name: Run Echo Ingest script
+        run: poetry run python util/realistic_data/ingest_echo_data.py
+        working-directory: ./operationsgateway-api
+
+      - name: Start API
+        run: nohup poetry run python -m operationsgateway_api.src.main > api-output.txt &
+        working-directory: ./operationsgateway-api
+
+      - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4
+        with:
+          node-version: 20
+          cache: 'yarn'
+
+      - name: Install dependencies
+        run: yarn --immutable
+
+      - name: Run playwright tests
+        run: yarn playwright:test:real
+
+      - name: Upload test report
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4
+        if: always()
+        with:
+          name: playwright-report-real-tests
+          path: playwright-report/
+          retention-days: 10

.gitignore

@@ -42,3 +42,4 @@ yarn-error.log*
 /test-results/
 /playwright-report/
 /playwright/.cache/
+/e2e/real/.auth

.yarn/sdks/eslint/bin/eslint.js

@@ -1,18 +1,25 @@
 #!/usr/bin/env node
 
 const {existsSync} = require(`fs`);
-const {createRequire} = require(`module`);
+const {createRequire, register} = require(`module`);
 const {resolve} = require(`path`);
+const {pathToFileURL} = require(`url`);
 
 const relPnpApiPath = "../../../../.pnp.cjs";
 
 const absPnpApiPath = resolve(__dirname, relPnpApiPath);
 const absRequire = createRequire(absPnpApiPath);
 
+const absPnpLoaderPath = resolve(absPnpApiPath, `../.pnp.loader.mjs`);
+const isPnpLoaderEnabled = existsSync(absPnpLoaderPath);
+
 if (existsSync(absPnpApiPath)) {
   if (!process.versions.pnp) {
     // Setup the environment to be able to require eslint/bin/eslint.js
     require(absPnpApiPath).setup();
+    if (isPnpLoaderEnabled && register) {
+      register(pathToFileURL(absPnpLoaderPath));
+    }
   }
 }
 

.yarn/sdks/eslint/lib/api.js

@@ -1,18 +1,25 @@
 #!/usr/bin/env node
 
 const {existsSync} = require(`fs`);
-const {createRequire} = require(`module`);
+const {createRequire, register} = require(`module`);
 const {resolve} = require(`path`);
+const {pathToFileURL} = require(`url`);
 
 const relPnpApiPath = "../../../../.pnp.cjs";
 
 const absPnpApiPath = resolve(__dirname, relPnpApiPath);
 const absRequire = createRequire(absPnpApiPath);
 
+const absPnpLoaderPath = resolve(absPnpApiPath, `../.pnp.loader.mjs`);
+const isPnpLoaderEnabled = existsSync(absPnpLoaderPath);
+
 if (existsSync(absPnpApiPath)) {
   if (!process.versions.pnp) {
     // Setup the environment to be able to require eslint
     require(absPnpApiPath).setup();
+    if (isPnpLoaderEnabled && register) {
+      register(pathToFileURL(absPnpLoaderPath));
+    }
   }
 }
 

.yarn/sdks/eslint/lib/unsupported-api.js

@@ -1,18 +1,25 @@
 #!/usr/bin/env node
 
 const {existsSync} = require(`fs`);
-const {createRequire} = require(`module`);
+const {createRequire, register} = require(`module`);
 const {resolve} = require(`path`);
+const {pathToFileURL} = require(`url`);
 
 const relPnpApiPath = "../../../../.pnp.cjs";
 
 const absPnpApiPath = resolve(__dirname, relPnpApiPath);
 const absRequire = createRequire(absPnpApiPath);
 
+const absPnpLoaderPath = resolve(absPnpApiPath, `../.pnp.loader.mjs`);
+const isPnpLoaderEnabled = existsSync(absPnpLoaderPath);
+
 if (existsSync(absPnpApiPath)) {
   if (!process.versions.pnp) {
     // Setup the environment to be able to require eslint/use-at-your-own-risk
     require(absPnpApiPath).setup();
+    if (isPnpLoaderEnabled && register) {
+      register(pathToFileURL(absPnpLoaderPath));
+    }
   }
 }
 

.yarn/sdks/prettier/bin/prettier.cjs

@@ -1,18 +1,25 @@
 #!/usr/bin/env node
 
 const {existsSync} = require(`fs`);
-const {createRequire} = require(`module`);
+const {createRequire, register} = require(`module`);
 const {resolve} = require(`path`);
+const {pathToFileURL} = require(`url`);
 
 const relPnpApiPath = "../../../../.pnp.cjs";
 
 const absPnpApiPath = resolve(__dirname, relPnpApiPath);
 const absRequire = createRequire(absPnpApiPath);
 
+const absPnpLoaderPath = resolve(absPnpApiPath, `../.pnp.loader.mjs`);
+const isPnpLoaderEnabled = existsSync(absPnpLoaderPath);
+
 if (existsSync(absPnpApiPath)) {
   if (!process.versions.pnp) {
     // Setup the environment to be able to require prettier/bin/prettier.cjs
     require(absPnpApiPath).setup();
+    if (isPnpLoaderEnabled && register) {
+      register(pathToFileURL(absPnpLoaderPath));
+    }
   }
 }
 

.yarn/sdks/prettier/index.cjs

@@ -1,18 +1,25 @@
 #!/usr/bin/env node
 
 const {existsSync} = require(`fs`);
-const {createRequire} = require(`module`);
+const {createRequire, register} = require(`module`);
 const {resolve} = require(`path`);
+const {pathToFileURL} = require(`url`);
 
 const relPnpApiPath = "../../../.pnp.cjs";
 
 const absPnpApiPath = resolve(__dirname, relPnpApiPath);
 const absRequire = createRequire(absPnpApiPath);
 
+const absPnpLoaderPath = resolve(absPnpApiPath, `../.pnp.loader.mjs`);
+const isPnpLoaderEnabled = existsSync(absPnpLoaderPath);
+
 if (existsSync(absPnpApiPath)) {
   if (!process.versions.pnp) {
     // Setup the environment to be able to require prettier
     require(absPnpApiPath).setup();
+    if (isPnpLoaderEnabled && register) {
+      register(pathToFileURL(absPnpLoaderPath));
+    }
   }
 }