From 092b2b4ce4162af8587eb004a95eac95827b7b58 Mon Sep 17 00:00:00 2001 From: Jacob Straszynski Date: Wed, 12 Mar 2025 21:21:45 -0700 Subject: [PATCH 1/3] Fixes for macOS + nested spire examples. Various small changes to make the nested-spire example work on a macOS host running Docker for Mac. --- On my system, the first run of scripts/set-env.sh yields: ``` 2025-03-13T02:46:39Z] bootstrapping root-agent. Error: connection error: desc = "transport: error while dialing: dial unix /tmp/spire-server/private/api.sock: connect: no such file or directory" ``` The naive thing to do is to merely re-run the script. If you re-run set-env.sh after the above failure, new certs are generated but the server appears to continue running and using the old ca. So lets just make it work the first time by waiting for the server. The other issues relate to Docker for Mac and it's hackish socket handling. The /var/run/docker.sock is apparently special cased somehow so we specify that single file in the volume mounts rather than the entire /var/run folder. --- The other error I saw had to do with sharing that `sharedSocket` folder: ``` "Agent crashed" error="unable to change UDS permissions: chmod /opt/spire/sockets/workload_api.sock: invalid argument" ``` This originates here https://github.com/spiffe/spire/blob/v1.5.1/pkg/agent/endpoints/endpoints_posix.go#L32 Again, docker for mac "breaks" sockets created in this shared volume when it's on the host filesystem because in practice it's some rube-goldberg contraption with a lightweight VM under the hood. So we just create a container-scoped volume instead, and bypass the macOS host filesystem entirely. This might make it a bit funky to interact with spire binaries running on our host - perhaps, but we could always just add some bastion container to the docker-compose.yml file if we want some tooling container for experimentation. Signed-off-by: Jacob Straszynski --- .../nested-spire/docker-compose.yaml | 22 ++++++++++++++----- .../nested-spire/scripts/set-env.sh | 19 ++++++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/docker-compose/nested-spire/docker-compose.yaml b/docker-compose/nested-spire/docker-compose.yaml index 8b4faa4..952a702 100644 --- a/docker-compose/nested-spire/docker-compose.yaml +++ b/docker-compose/nested-spire/docker-compose.yaml @@ -15,9 +15,9 @@ services: hostname: root-agent volumes: # Share root-agent socket to be accessed by nested servers - - ./sharedRootSocket:/opt/spire/sockets + - spire-sockets:/opt/spire/sockets - ./root/agent:/opt/spire/conf/agent - - /var/run/:/var/run/ + - /var/run/docker.sock:/var/run/docker.sock command: ["-config", "/opt/spire/conf/agent/agent.conf"] # NestedA nestedA-server: @@ -31,7 +31,7 @@ services: depends_on: ["root-server","root-agent"] volumes: # Add root-agent socket - - ./sharedRootSocket:/opt/spire/sockets + - spire-sockets:/opt/spire/sockets - ./nestedA/server:/opt/spire/conf/server command: ["-config", "/opt/spire/conf/server/server.conf"] nestedA-agent: @@ -40,7 +40,7 @@ services: depends_on: ["nestedA-server"] volumes: - ./nestedA/agent:/opt/spire/conf/agent - - /var/run/:/var/run/ + - /var/run/docker.sock:/var/run/docker.sock command: ["-config", "/opt/spire/conf/agent/agent.conf"] nestedB-server: # Share the host pid namespace so this server can be attested by the root agent @@ -53,7 +53,7 @@ services: - org.example.name=nestedB-server volumes: # Add root-agent socket - - ./sharedRootSocket:/opt/spire/sockets + - spire-sockets:/opt/spire/sockets - ./nestedB/server:/opt/spire/conf/server command: ["-config", "/opt/spire/conf/server/server.conf"] nestedB-agent: @@ -62,5 +62,15 @@ services: depends_on: ["nestedB-server"] volumes: - ./nestedB/agent:/opt/spire/conf/agent - - /var/run/:/var/run/ + - /var/run/docker.sock:/var/run/docker.sock command: ["-config", "/opt/spire/conf/agent/agent.conf"] + +volumes: + # Allows macs to have functional domain sockets between spire agents and + # servers by avoiding a volume mount to the host filesystem and instead + # creating a tmpfs volume to be used as a socket directory. + spire-sockets: + driver: local + driver_opts: + type: tmpfs + device: tmpfs diff --git a/docker-compose/nested-spire/scripts/set-env.sh b/docker-compose/nested-spire/scripts/set-env.sh index df2ff98..e16dc4e 100755 --- a/docker-compose/nested-spire/scripts/set-env.sh +++ b/docker-compose/nested-spire/scripts/set-env.sh @@ -31,6 +31,20 @@ fingerprint() { openssl x509 -in "$1" -outform DER | openssl sha1 -r | awk '{print $1}' } +wait-for-server() { + for ((i=1;i<=30;i++)); do + if ! docker-compose -f "${PARENT_DIR}"/docker-compose.yaml exec -T $1 /opt/spire/bin/spire-server healthcheck; then + log "Waiting for $1 to start..." + sleep 1 + else + log "${green}$1 is up and running.${norm}" + return 0 + fi + done + log "${red}$1 did not start.${norm}" + return 1 +} + check-entry-is-propagated() { # Check at most 30 times that the agent has successfully synced down the workload entry. # Wait one second between checks. @@ -68,6 +82,9 @@ setup "${PARENT_DIR}"/root/server "${PARENT_DIR}"/root/agent log "Start root server" docker-compose -f "${PARENT_DIR}"/docker-compose.yaml up -d root-server +wait-for-server root-server +docker-compose -f "${PARENT_DIR}"/docker-compose.yaml exec -T root-server /opt/spire/bin/spire-server bundle show > "${PARENT_DIR}"/root/agent/bootstrap.crt + log "bootstrapping root-agent." docker-compose -f "${PARENT_DIR}"/docker-compose.yaml exec -T root-server /opt/spire/bin/spire-server bundle show > "${PARENT_DIR}"/root/agent/bootstrap.crt @@ -103,6 +120,7 @@ setup "${PARENT_DIR}"/nestedA/server "${PARENT_DIR}"/nestedA/agent log "Starting nestedA-server.." docker-compose -f "${PARENT_DIR}"/docker-compose.yaml up -d nestedA-server +wait-for-server nestedA-server log "bootstrapping nestedA agent..." docker-compose -f "${PARENT_DIR}"/docker-compose.yaml exec -T nestedA-server /opt/spire/bin/spire-server bundle show > "${PARENT_DIR}"/nestedA/agent/bootstrap.crt @@ -117,6 +135,7 @@ setup "${PARENT_DIR}"/nestedB/server "${PARENT_DIR}"/nestedB/agent log "Starting nestedB-server.." docker-compose -f "${PARENT_DIR}"/docker-compose.yaml up -d nestedB-server +wait-for-server nestedB-server log "bootstrapping nestedB agent..." docker-compose -f "${PARENT_DIR}"/docker-compose.yaml exec -T nestedB-server /opt/spire/bin/spire-server bundle show > "${PARENT_DIR}"/nestedB/agent/bootstrap.crt From fa3bb3d26ccb6eb3569102e13b171db72931b617 Mon Sep 17 00:00:00 2001 From: Jacob Straszynski Date: Wed, 12 Mar 2025 23:02:26 -0700 Subject: [PATCH 2/3] bump images in nested spire example Signed-off-by: Jacob Straszynski --- docker-compose/nested-spire/docker-compose.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose/nested-spire/docker-compose.yaml b/docker-compose/nested-spire/docker-compose.yaml index 952a702..4c70709 100644 --- a/docker-compose/nested-spire/docker-compose.yaml +++ b/docker-compose/nested-spire/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3' services: # Root root-server: - image: ghcr.io/spiffe/spire-server:1.5.1 + image: ghcr.io/spiffe/spire-server:1.11.2 hostname: root-server volumes: - ./root/server:/opt/spire/conf/server @@ -10,7 +10,7 @@ services: root-agent: # Share the host pid namespace so this agent can attest the nested servers pid: "host" - image: ghcr.io/spiffe/spire-agent:1.5.1 + image: ghcr.io/spiffe/spire-agent:1.11.2 depends_on: ["root-server"] hostname: root-agent volumes: @@ -23,7 +23,7 @@ services: nestedA-server: # Share the host pid namespace so this server can be attested by the root agent pid: "host" - image: ghcr.io/spiffe/spire-server:1.5.1 + image: ghcr.io/spiffe/spire-server:1.11.2 hostname: nestedA-server labels: # label to attest server against root-agent @@ -35,7 +35,7 @@ services: - ./nestedA/server:/opt/spire/conf/server command: ["-config", "/opt/spire/conf/server/server.conf"] nestedA-agent: - image: ghcr.io/spiffe/spire-agent:1.5.1 + image: ghcr.io/spiffe/spire-agent:1.11.2 hostname: nestedA-agent depends_on: ["nestedA-server"] volumes: @@ -45,7 +45,7 @@ services: nestedB-server: # Share the host pid namespace so this server can be attested by the root agent pid: "host" - image: ghcr.io/spiffe/spire-server:1.5.1 + image: ghcr.io/spiffe/spire-server:1.11.2 hostname: nestedB-server depends_on: ["root-server","root-agent"] labels: @@ -57,7 +57,7 @@ services: - ./nestedB/server:/opt/spire/conf/server command: ["-config", "/opt/spire/conf/server/server.conf"] nestedB-agent: - image: ghcr.io/spiffe/spire-agent:1.5.1 + image: ghcr.io/spiffe/spire-agent:1.11.2 hostname: nestedB-agent depends_on: ["nestedB-server"] volumes: From f1059fd5b1505e0e07417735db21416e5a640708 Mon Sep 17 00:00:00 2001 From: Jacob Straszynski Date: Wed, 12 Mar 2025 23:18:37 -0700 Subject: [PATCH 3/3] update docs and drop the sharedRootSocket Signed-off-by: Jacob Straszynski --- docker-compose/nested-spire/README.md | 18 +++++++++--------- docker-compose/nested-spire/scripts/set-env.sh | 4 ---- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/docker-compose/nested-spire/README.md b/docker-compose/nested-spire/README.md index 626a228..ebd3a2a 100644 --- a/docker-compose/nested-spire/README.md +++ b/docker-compose/nested-spire/README.md @@ -34,21 +34,21 @@ Before proceeding, review the following system requirements: This tutorial's `nested-spire` main directory contains three subdirectories, one for each of the SPIRE deployments: `root`, `nestedA` and `nestedB`. These directories hold the configuration files for the SPIRE Servers and Agents. They will also contain the private keys and certificates created to attest the Agents on the Servers with the [x509pop Node Attestor](https://github.com/spiffe/spire/blob/main/doc/plugin_server_nodeattestor_x509pop.md) plugin. Private keys and certificates are created at the initialization of the scenario using a Go application, the details of which are out of the scope of this tutorial. -## Create a Shared Directory +## The Shared Socket Directory -The first thing needed is a local directory that will be volume mounted on the services to share the Workload API between the root SPIRE Agent and its nested SPIRE Servers. This tutorial uses `.../spire-tutorials/docker-compose/nested-spire/sharedRootSocket` as the shared directory. +We use a shared directory that will be volume mounted on the services to share the Workload API between the root SPIRE Agent and its nested SPIRE Servers. This tutorial uses a named volume in its docker-compose configuration to share the Workload API socket. ## Configuring Root SPIRE Deployment Configuration files for [root-server](root/server/server.conf) and [root-agent](root/agent/agent.conf) have not been changed from the default `server.conf` and `agent.conf` files, but it's worth noting the location defined to bind the workload API socket by the SPIRE Agent: `socket_path ="/opt/spire/sockets/workload_api.sock"`. This path will be used later to configure a volume to share the Workload API with the nested SPIRE Servers. -We define all the services for the tutorial in the [docker-compose.yaml](docker-compose.yaml) file. In the `root-agent` service definition we mount the `/opt/spire/sockets` directory from the SPIRE Agent container on the new local directory `sharedRootSocket`. In the next section, when defining the nested SPIRE Server services, we'll use this directory to mount the `root-agent` socket on the SPIRE Server containers. +We define all the services for the tutorial in the [docker-compose.yaml](docker-compose.yaml) file. In the `root-agent` service definition we mount the `/opt/spire/sockets` directory from the SPIRE Agent container on the named `spire-sockets` volume. In the next section, when defining the nested SPIRE Server services, we'll use this directory to mount the `root-agent` socket on the SPIRE Server containers. ```console services: # Root root-server: - image: ghcr.io/spiffe/spire-server:1.5.1 + image: ghcr.io/spiffe/spire-server:1.11.2 hostname: root-server volumes: - ./root/server:/opt/spire/conf/server @@ -56,12 +56,12 @@ We define all the services for the tutorial in the [docker-compose.yaml](docker- root-agent: # Share the host pid namespace so this agent can attest the nested servers pid: "host" - image: ghcr.io/spiffe/spire-agent:1.5.1 + image: ghcr.io/spiffe/spire-agent:1.11.2 depends_on: ["root-server"] hostname: root-agent volumes: # Share root agent socket to be accessed by nestedA and nestedB servers - - ./sharedRootSocket:/opt/spire/sockets + - spire-sockets:/opt/spire/sockets - ./root/agent:/opt/spire/conf/agent - /var/run/:/var/run/ command: ["-config", "/opt/spire/conf/agent/agent.conf"] @@ -85,7 +85,7 @@ The configuration file for the [nestedA-server](./nestedA/server/server.conf) in } ``` -The Docker Compose definition for the `nestedA-server` service in the [docker-compose.yaml](docker-compose.yaml) file mounts the new local directory `sharedRootSocket` as a volume. Remember from the previous section that the `root-agent` socket is mounted on that directory. That way the `nestedA-server` can access the `root-agent` workload API and fetch its SVID. +The Docker Compose definition for the `nestedA-server` service in the [docker-compose.yaml](docker-compose.yaml) file mounts the `spire-sockets` named volume. Remember from the previous section that the `root-agent` socket is mounted on that directory. That way the `nestedA-server` can access the `root-agent` workload API and fetch its SVID. ```console nestedA-server: @@ -97,8 +97,8 @@ The Docker Compose definition for the `nestedA-server` service in the [docker-co # label to attest nestedA-server against root-agent - org.example.name=nestedA volumes: - # Add root agent socket - - ./shared/rootSocket:/opt/spire/sockets + # Add root agent socket + - spire-sockets:/opt/spire/sockets - ./nestedA/server:/opt/spire/conf/server command: ["-config", "/opt/spire/conf/server/server.conf"] ``` diff --git a/docker-compose/nested-spire/scripts/set-env.sh b/docker-compose/nested-spire/scripts/set-env.sh index e16dc4e..49f1d70 100755 --- a/docker-compose/nested-spire/scripts/set-env.sh +++ b/docker-compose/nested-spire/scripts/set-env.sh @@ -71,10 +71,6 @@ fi sed -i.bak "s#\#container_id_cgroup_matchers#container_id_cgroup_matchers#" "${PARENT_DIR}"/root/agent/agent.conf sed -i.bak "s#CGROUP_MATCHERS#$CGROUP_MATCHERS#" "${PARENT_DIR}"/root/agent/agent.conf -# create a shared folder for root agent socket to be accessed by nestedA and nestedB servers -mkdir -p "${PARENT_DIR}"/sharedRootSocket - - # Starts root SPIRE deployment log "Generate certificates for the root SPIRE deployment" setup "${PARENT_DIR}"/root/server "${PARENT_DIR}"/root/agent