Add load testing utility (#2810)

* Add load testing utility

* Add k6 load testing set up

* Update scripts

* Use localhost URL for loadtesting

This should generally be used for testing capacity locally, not in a
live distribution. The latter stresses the actual servers and can lead
to your IP being blocked.

* Indicate load testing from HACKING.md

Author: shonfeder

HACKING.md

@@ -59,12 +59,19 @@ This will restart the server on filesystem changes.
 
 ### Running Tests
 
+#### Unit tests
+
 You can run the unit test suite with:
 
 ```bash
 make test
 ```
 
+#### Load tests
+
+See the readme's for running load tests via [k6](./test/load-test/k6/README.md)
+or [locust](./test/load-test/locust/README.md).
+
 ### Building the Playground
 
 The OCaml Playground is compiled separately from the rest of the server. The generated assets can be found in

test/load-test/k6/.gitignore

@@ -0,0 +1 @@
+_results/

test/load-test/k6/README.md

@@ -0,0 +1,45 @@
+# k6 Load Tests
+
+This directory contains a [k6](https://grafana.com/docs/k6/latest/) script for
+load testing ocaml.org endpoints.
+
+- The k6 test script is defined in [./script.js](./script.js). 
+- The runner script [./run.sh](./run.sh) will run `./script.js` with the k6
+  docker image.
+  
+
+## Running load tests
+
+### Prerequisites
+
+- docker
+- k6parser
+- xdg-open (MacOS users may need to adjust the runner script)
+
+### Usage
+
+``` sh
+./run.sh script.js
+```
+
+will run a load test against all defined endpoints using 10 virtual users, and
+running for 30 seconds.
+
+You can specify different values for the concurrent users and duration using the
+`--vus` and `--duration` flags, respectively. E.g., to test with 100 virtual
+users for one minute:
+
+``` sh
+./run.sh --vus 100 --duration 1m script.js
+```
+
+## Reviewing the results
+
+k6 will print out the a summary of results when the test is finished. See
+<https://grafana.com/docs/k6/latest/get-started/results-output/> for
+documentation on how to interpret the results.
+
+- Detailed results are written to a gzipped JOSN file
+  `_results/{day}T{time}-results.gz`.
+- A browser will be opened with a visualization plotting general responsiveness,
+  loaded with the file `_results/{day}T{time}-results.report`.

test/load-test/k6/run.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env sh
+
+# Ignore posix compliance errors in shellcheck
+#shellcheck disable=SC3000-SC3061
+
+set -eu
+set -o pipefail
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+RESULTS_DIR=_results
+USER_ID=$(id -u) # ensure we can write to the mounted dir
+TIME=$(date +%FT%T)
+RESULTS="${RESULTS_DIR}/${TIME}-results.gz"
+GRAPH="${RESULTS_DIR}/${TIME}-report.html"
+
+
+mkdir -p "$RESULTS_DIR"
+
+docker run --rm \
+    -u "$USER_ID" \
+    -v "$SCRIPT_DIR":/home/k6 grafana/k6 \
+    --out json="$RESULTS" \
+    "$@"
+
+k6parser "$RESULTS" --output "$GRAPH"
+
+xdg-open "$GRAPH"

test/load-test/k6/script.js

@@ -0,0 +1,88 @@
+import http from "k6/http";
+import { sleep } from "k6";
+
+export const options = {
+  // A number specifying the number of VUs to run concurrently.
+  vus: 10,
+  // A string specifying the total duration of the test run.
+  duration: "30s",
+
+  // The following section contains configuration options for execution of this
+  // test script in Grafana Cloud.
+  //
+  // See https://grafana.com/docs/grafana-cloud/k6/get-started/run-cloud-tests-from-the-cli/
+  // to learn about authoring and running k6 test scripts in Grafana k6 Cloud.
+  //
+  // cloud: {
+  //   // The ID of the project to which the test is assigned in the k6 Cloud UI.
+  //   // By default tests are executed in default project.
+  //   projectID: "",
+  //   // The name of the test in the k6 Cloud UI.
+  //   // Test runs with the same name will be grouped.
+  //   name: "script.js"
+  // },
+
+  // Uncomment this section to enable the use of Browser API in your tests.
+  //
+  // See https://grafana.com/docs/k6/latest/using-k6-browser/running-browser-tests/ to learn more
+  // about using Browser API in your test scripts.
+  //
+  // scenarios: {
+  //   // The scenario name appears in the result summary, tags, and so on.
+  //   // You can give the scenario any name, as long as each name in the script is unique.
+  //   ui: {
+  //     // Executor is a mandatory parameter for browser-based tests.
+  //     // Shared iterations in this case tells k6 to reuse VUs to execute iterations.
+  //     //
+  //     // See https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/ for other executor types.
+  //     executor: 'shared-iterations',
+  //     options: {
+  //       browser: {
+  //         // This is a mandatory parameter that instructs k6 to launch and
+  //         // connect to a chromium-based browser, and use it to run UI-based
+  //         // tests.
+  //         type: 'chromium',
+  //       },
+  //     },
+  //   },
+  // }
+};
+
+// The function that defines VU logic.
+//
+// See https://grafana.com/docs/k6/latest/examples/get-started-with-k6/ to learn more
+// about authoring k6 scripts.
+
+const base_url = "http://localhost:8080";
+
+function endpoint(u) {
+  return base_url + u;
+}
+
+export default function () {
+  // Landing page
+  http.get(endpoint("/"));
+
+  // Core doc page
+  http.get(endpoint("/p/core/latest/doc/index.html"));
+
+  // Top level pages
+  http.get(endpoint("/install"));
+  http.get(endpoint("/docs/tour-of-ocaml"));
+  http.get(endpoint("/docs"));
+  http.get(endpoint("/platform"));
+  http.get(endpoint("/packages"));
+  http.get(endpoint("/community"));
+  http.get(endpoint("/changelog"));
+  http.get(endpoint("/play"));
+  http.get(endpoint("/industrial-users"));
+  http.get(endpoint("/academic-users"));
+
+  // some package searches
+  // Grouping the urls, see https://grafana.com/docs/k6/latest/using-k6/http-requests/#url-grouping
+  const package_search_tag = { tags: { name: "PacakageSearch" } };
+  ["http", "server", "cli", "core", "eio", "graph"].forEach((q) => {
+    http.get(endpoint(`/packages/autocomplete?q=${q}`), package_search_tag);
+    http.get(endpoint(`/packages/search?q=${q}`), package_search_tag);
+  });
+}

test/load-test/locust/.gitignore

@@ -0,0 +1 @@
+__pycache__/

test/load-test/locust/README.md

@@ -0,0 +1,40 @@
+# Locust Load Tests
+
+This directory contains a [locust](https://locust.io/) script for load testing
+ocaml.org endpoints.
+
+## Running simple load tests
+
+1. Start the test framework running with
+
+   ``` sh
+   ./main.sh
+   ```
+
+2. Navigate to http://0.0.0.0:8089
+
+3. Configure
+  - the max number of users to simulate
+  - the number of new users to add to the simulation every second
+  - the host (e.g., `http://localhost:8080`)
+
+## Running load tests with multiple cores
+
+``` sh
+./main.sh n
+```
+
+where `n` is the number of processes to run concurrently.
+
+## Reviewing the load tests
+
+- Click "Stop" when you are finished running your test.
+- Review the various tabs, or click "Download data" for options to download the
+  test results.
+- You can also review prometheus metrics about the staging and prod servers at
+  https://status.ocaml.ci.dev/d/be358r0z9ai9sf/ocaml-org
+
+## Adding new routines
+
+Tests are defined as "tasks" (sequences of site traversal) in
+[./locustfile.py](./locustfile.py).

test/load-test/locust/locustfile.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+#
+import random
+from locust import FastHttpUser, task, between
+
+# obtained via https://random-word-api.herokuapp.com/word?number=200
+words = ["sacraria","seedcake","philtered","leadiest","cloverleafs","snaffle","lyrisms","ankhs","bedtimes","carrier","restabling","playlets","occupation","overreport","printers","scintigraphic","spa","corsages","enwound","gossipmonger","hydragog","waterlily","avulsions","sonneteerings","spilt","hemocytes","tamandus","rais","minaret","coliseums","ultramicrotome","attribute","phosphite","scincoids","scooting","frightfulnesses","carbamides","sculpture","irresponsive","overtasks","expertise","weet","consociations","tulles","hared","pigginesses","oversimple","theologs","adverb","inamoratas","teeny","rapacities","assonant","metestrus","cyanohydrin","smiting","polychete","merest","tautological","phyllopod","petahertz","plainspokenness","pavior","penitently","omikrons","cigarlike","foetal","diebacks","downlight","kinship","warmish","titleholders","suppositions","resuscitations","tiffing","outsung","homed","alternated","cranks","piaster","allotters","nonirradiated","protohistories","finned","decouple","shahs","foeman","perfidious","soarers","thoroughpins","gastrulating","thrivers","convention","roughened","uncircumcised","clabbering","leadscrew","panfuls","nilgais","evolver","overvoting","furrower","ichthyosaurs","internalizes","borschts","regrouping","lordlier","roguish","microseismicity","besmiling","mattoids","cholerically","fibrosarcomas","farinha","curricles","triradiate","beringed","electrolysis","kashmirs","dirdums","ignorami","otalgic","lusciously","blotty","pizzaz","educe","pendant","disposable","autolyzes","outjutting","interfused","operagoers","fustian","theretofore","dean","unsullied","goitrogenic","ultrasafe","potboil","geochemistries","outdesigned","ephedras","woodlore","illuminatingly","guardrooms","sheldrakes","leachable","theistically","reconception","beachboys","recriminates","almuds","changeabilities","flareups","machinate","verbalizations","dendrologist","unkept","copulatives","restyles","parceled","caecilians","mortgager","thunderstones","labarums","wiliness","hydroplane","already","unlatch","swineherds","alternate","whodunit","hoodiest","sainted","detract","inspiring","fantastically","macaws","adsorbs","thickets","blogs","greenfields","ariettes","camphor","hornpipe","uninventive","boatyard","boomiest","lollingly","congresspersons","painter","radiocarbons","impiously","unfeigned","matchlocks","screwballs","stickies","muddlers","resentful","meats"]
+
+class OcamlOrgUsere(FastHttpUser):
+    wait_time = between(1, 5) # range of seconds a user waits between clicks
+
+    @task
+    def landing(self):
+        self.client.get("/")
+        self.client.get("/install")
+        self.client.get("/p/core/latest/doc/index.html")
+        self.client.get("/docs/tour-of-ocaml")
+
+    @task
+    def top_level_pages(self):
+        self.client.get("/docs")
+        self.client.get("/platform")
+        self.client.get("/packages")
+        self.client.get("/community")
+        self.client.get("/changelog")
+        self.client.get("/play")
+        self.client.get("/industrial-users")
+        self.client.get("/academic-users")
+
+    @task
+    def package_searches(self):
+        query = random.choice(words)
+        self.client.get(f"/packages/autocomplete?q={query}")
+        self.client.get(f"/packages/search?q={query}")

test/load-test/locust/main.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env sh
+
+# Ignore posix compliance errors in shellcheck
+#shellcheck disable=SC3000-SC3061
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# see https://docs.locust.io/en/stable/running-distributed.html#distributed-load-generation
+n_procs="$1"
+
+docker run -p 8089:8089 -v "$SCRIPT_DIR":/mnt/locust locustio/locust \
+    --locustfile /mnt/locust/locustfile.py \
+    ${n_procs:+"--processes=${n_procs}"} # Build the --processes flag if n_procs is not nil