From 77fb889362314eea5b95900f934d3e3165019262 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 3 Mar 2023 12:48:08 -0500
Subject: [PATCH 01/22] Package all globzl zone tarballs into single tarball

---
 .github/buildomat/jobs/package.sh | 43 ++++++++++++++++++++++---------
 1 file changed, 31 insertions(+), 12 deletions(-)

diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
index 79250e91152..0571d1b7687 100644
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -5,10 +5,14 @@
 #: target = "helios-latest"
 #: rust_toolchain = "1.66.1"
 #: output_rules = [
-#:	"=/work/package.tar.gz",
+#:	"=/work/global-zone-packages.tar.gz",
 #:	"=/work/zones/*.tar.gz",
 #: ]
 #:
+#: [[publish]]
+#: series = "image"
+#: name = "global-zone-packages"
+#: from_output = "/out/global-zone-packages.tar.gz"
 
 set -o errexit
 set -o pipefail
@@ -17,18 +21,33 @@ set -o xtrace
 cargo --version
 rustc --version
 
+# Build
 ptime -m ./tools/install_builder_prerequisites.sh -yp
-ptime -m ./tools/create_self_signed_cert.sh -yp
-
 ptime -m cargo run --locked --release --bin omicron-package -- package
 
-files=(
-	out/*.tar
-	package-manifest.toml
-	smf/sled-agent/config.toml
-	target/release/omicron-package
-	tools/create_virtual_hardware.sh
-)
-ptime -m tar cvzf /work/package.tar.gz "${files[@]}"
+# Assemble global zone files in a temporary directory.
+tmp=$(mktemp -d)
+mkdir -p "${tmp}/sled-agent"
+tar -xvzf out/omicron-sled-agent.tar -C "${tmp}/sled-agent"
+mkdir -p "${tmp}/maghemite"
+tar -xvzf out/maghemite.tar -C "${tmp}/maghemite"
+
+# Load those global zone files into a tarball that's ready to be exported.
+mkdir -p /work
+ptime -m tar cvzf /work/global-zone-packages.tar.gz -C "${tmp}" .
+
+# Assemble Zone Images into their respective output locations.
 mkdir -p /work/zones
-mv out/*.tar.gz /work/zones/
+zones=(
+	out/clickhouse.tar.gz
+	out/cockroachdb.tar.gz
+	out/crucible-pantry.tar.gz
+	out/crucible.tar.gz
+	out/external-dns.tar.gz
+	out/internal-dns.tar.gz
+	out/omicron-nexus.tar.gz
+	out/oximeter-collector.tar.gz
+	out/propolis-server.tar.gz
+	out/switch-asic.tar.gz
+)
+mv "${zones[@]}" /work/zones/

From eced651d3f6b1b8d990a77c88ae064d35bbc731f Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 3 Mar 2023 13:26:45 -0500
Subject: [PATCH 02/22] switch variant asic

---
 .github/buildomat/jobs/package.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
index 0571d1b7687..1e179cc62cc 100644
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -23,7 +23,7 @@ rustc --version
 
 # Build
 ptime -m ./tools/install_builder_prerequisites.sh -yp
-ptime -m cargo run --locked --release --bin omicron-package -- package
+ptime -m cargo run --locked --release --bin omicron-package -- -t switch_variant=asic package
 
 # Assemble global zone files in a temporary directory.
 tmp=$(mktemp -d)

From 170ee348002f15b0357e11e4249352739e9f6b08 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 3 Mar 2023 14:05:58 -0500
Subject: [PATCH 03/22] update name of global zone packages

---
 .github/buildomat/jobs/deploy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index 6217e33d794..885f3ea33ff 100644
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -109,7 +109,7 @@ pfexec mkdir /opt/oxide/work
 pfexec chown build:build /opt/oxide/work
 cd /opt/oxide/work
 
-ptime -m tar xvzf /input/package/work/package.tar.gz
+ptime -m tar xvzf /input/package/work/global-zone-packages.tar.gz
 cp /input/package/work/zones/* out/
 mkdir tests
 for p in /input/build-end-to-end-tests/work/*.gz; do

From 2b63d56991041d62f279cafd00419363b7b9c9c7 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 3 Mar 2023 14:50:00 -0500
Subject: [PATCH 04/22] More attempts to correctly use this assembled GZ
 package

---
 .github/buildomat/jobs/deploy.sh | 24 ++++++------------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index 885f3ea33ff..11ed6a369b1 100644
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -109,8 +109,10 @@ pfexec mkdir /opt/oxide/work
 pfexec chown build:build /opt/oxide/work
 cd /opt/oxide/work
 
-ptime -m tar xvzf /input/package/work/global-zone-packages.tar.gz
+mkdir out/
+ptime -m tar xvzf /input/package/work/global-zone-packages.tar.gz -C out
 cp /input/package/work/zones/* out/
+
 mkdir tests
 for p in /input/build-end-to-end-tests/work/*.gz; do
 	ptime -m gunzip < "$p" > "tests/$(basename "${p%.gz}")"
@@ -144,27 +146,13 @@ pfexec svccfg import /var/svc/manifest/site/tcpproxy.xml
 #
 pfexec ipadm create-addr -T static -a 192.168.1.199/24 igb0/sidehatch
 
-#
-# Modify config-rss.toml in the sled-agent zone to use our system's IP and MAC
+# Modify config-rss.toml in the sled-agent to use our system's IP and MAC
 # address for upstream connectivity.
-#
-tar xf out/omicron-sled-agent.tar pkg/config-rss.toml
 sed -e 's/^# address =.*$/address = "192.168.1.199"/' \
 	-e "s/^mac =.*$/mac = \"$(dladm show-phys -m -p -o ADDRESS | head -n 1)\"/" \
-	-i pkg/config-rss.toml
-tar rf out/omicron-sled-agent.tar pkg/config-rss.toml
-rm -rf pkg
+	-i out/sled-agent/pkg/config-rss.toml
 
-#
-# This OMICRON_NO_UNINSTALL hack here is so that there is no implicit uninstall
-# before the install.  This doesn't work right now because, above, we made
-# /var/oxide a file system so you can't remove it (EBUSY) like a regular
-# directory.  The lab-netdev target is a ramdisk system that is always cleared
-# out between runs, so it has not had any state yet that requires
-# uninstallation.
-#
-OMICRON_NO_UNINSTALL=1 \
-    ptime -m pfexec ./target/release/omicron-package install
+svccfg import out/sled-agent/pkg/manifest.xml
 
 ./tests/bootstrap
 rm ./tests/bootstrap

From 748b0e93d1dcfe3945a2e9e6943bd8a22f2b11be Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 3 Mar 2023 16:22:08 -0500
Subject: [PATCH 05/22] try again with supplementary package

---
 .github/buildomat/jobs/deploy.sh  |  1 +
 .github/buildomat/jobs/package.sh | 12 ++++++++++++
 2 files changed, 13 insertions(+)

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index 11ed6a369b1..d28d9f52a7a 100644
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -109,6 +109,7 @@ pfexec mkdir /opt/oxide/work
 pfexec chown build:build /opt/oxide/work
 cd /opt/oxide/work
 
+ptime -m tar xvzf /input/package/work/utilities-package.tar.gz
 mkdir out/
 ptime -m tar xvzf /input/package/work/global-zone-packages.tar.gz -C out
 cp /input/package/work/zones/* out/
diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
index 1e179cc62cc..50e815f55ca 100644
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -5,6 +5,7 @@
 #: target = "helios-latest"
 #: rust_toolchain = "1.66.1"
 #: output_rules = [
+#:	"=/work/utilities-package.tar.gz",
 #:	"=/work/global-zone-packages.tar.gz",
 #:	"=/work/zones/*.tar.gz",
 #: ]
@@ -25,6 +26,17 @@ rustc --version
 ptime -m ./tools/install_builder_prerequisites.sh -yp
 ptime -m cargo run --locked --release --bin omicron-package -- -t switch_variant=asic package
 
+
+# Assemble some utilities into a tarball that can be used by deployment
+# phases of buildomat.
+
+utilities=(
+  package-manifest.toml
+  tools/create_virtual_hardware.sh
+)
+
+ptime -m tar cvzf /work/utilities-package.tar.gz "${utilities[@]}"
+
 # Assemble global zone files in a temporary directory.
 tmp=$(mktemp -d)
 mkdir -p "${tmp}/sled-agent"

From 3d34f05c16a96c31e49c88948cd20a60b64b0a76 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 3 Mar 2023 17:13:43 -0500
Subject: [PATCH 06/22] /opt/oxide paths

---
 .github/buildomat/jobs/deploy.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index d28d9f52a7a..b37521054b2 100644
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -153,7 +153,9 @@ sed -e 's/^# address =.*$/address = "192.168.1.199"/' \
 	-e "s/^mac =.*$/mac = \"$(dladm show-phys -m -p -o ADDRESS | head -n 1)\"/" \
 	-i out/sled-agent/pkg/config-rss.toml
 
-svccfg import out/sled-agent/pkg/manifest.xml
+mkdir -p /opt/oxide
+mv out/sled-agent /opt/oxide/
+svccfg import /opt/oxide/sled-agent/pkg/manifest.xml
 
 ./tests/bootstrap
 rm ./tests/bootstrap

From dcc9f6eaea84795ad2e6c36d1c59368a9ffc17b1 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 3 Mar 2023 19:44:43 -0500
Subject: [PATCH 07/22] Pfexec

---
 .github/buildomat/jobs/deploy.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index b37521054b2..5863588bdd1 100644
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -153,9 +153,8 @@ sed -e 's/^# address =.*$/address = "192.168.1.199"/' \
 	-e "s/^mac =.*$/mac = \"$(dladm show-phys -m -p -o ADDRESS | head -n 1)\"/" \
 	-i out/sled-agent/pkg/config-rss.toml
 
-mkdir -p /opt/oxide
-mv out/sled-agent /opt/oxide/
-svccfg import /opt/oxide/sled-agent/pkg/manifest.xml
+pfexec mv out/sled-agent /opt/oxide/
+pfexec svccfg import /opt/oxide/sled-agent/pkg/manifest.xml
 
 ./tests/bootstrap
 rm ./tests/bootstrap

From 6e836016e6ab92eaf889610040bba742c8b61866 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 3 Mar 2023 20:51:29 -0500
Subject: [PATCH 08/22] I forgot maghemite

---
 .github/buildomat/jobs/deploy.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index 5863588bdd1..6226bdcd5fd 100644
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -154,6 +154,7 @@ sed -e 's/^# address =.*$/address = "192.168.1.199"/' \
 	-i out/sled-agent/pkg/config-rss.toml
 
 pfexec mv out/sled-agent /opt/oxide/
+pfexec mv out/maghemite /opt/oxide/
 pfexec svccfg import /opt/oxide/sled-agent/pkg/manifest.xml
 
 ./tests/bootstrap

From 0a195c14d3a7f3d58addde591919ce799b59141e Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 3 Mar 2023 21:40:13 -0500
Subject: [PATCH 09/22] jk it's called mg-ddm

---
 .github/buildomat/jobs/deploy.sh  | 2 +-
 .github/buildomat/jobs/package.sh | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index 6226bdcd5fd..fc1e1b5b540 100644
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -154,7 +154,7 @@ sed -e 's/^# address =.*$/address = "192.168.1.199"/' \
 	-i out/sled-agent/pkg/config-rss.toml
 
 pfexec mv out/sled-agent /opt/oxide/
-pfexec mv out/maghemite /opt/oxide/
+pfexec mv out/mg-ddm /opt/oxide/
 pfexec svccfg import /opt/oxide/sled-agent/pkg/manifest.xml
 
 ./tests/bootstrap
diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
index 50e815f55ca..90ad4a1950e 100644
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -41,8 +41,8 @@ ptime -m tar cvzf /work/utilities-package.tar.gz "${utilities[@]}"
 tmp=$(mktemp -d)
 mkdir -p "${tmp}/sled-agent"
 tar -xvzf out/omicron-sled-agent.tar -C "${tmp}/sled-agent"
-mkdir -p "${tmp}/maghemite"
-tar -xvzf out/maghemite.tar -C "${tmp}/maghemite"
+mkdir -p "${tmp}/mg-ddm"
+tar -xvzf out/maghemite.tar -C "${tmp}/mg-ddm"
 
 # Load those global zone files into a tarball that's ready to be exported.
 mkdir -p /work

From 4c7127a6cb53ec609422542e2bcbaafcf000f85d Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 3 Mar 2023 22:33:04 -0500
Subject: [PATCH 10/22] ... are config files load-bearing to create virtual hw?

---
 .github/buildomat/jobs/package.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
index 90ad4a1950e..6a76d09b269 100644
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -32,6 +32,7 @@ ptime -m cargo run --locked --release --bin omicron-package -- -t switch_variant
 
 utilities=(
   package-manifest.toml
+  mf/sled-agent/config.toml
   tools/create_virtual_hardware.sh
 )
 

From 8532c7eeb838662e4e5ed764421e682e3b89fe73 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Mon, 6 Mar 2023 10:07:57 -0500
Subject: [PATCH 11/22] fix typo

---
 .github/buildomat/jobs/package.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 mode change 100644 => 100755 .github/buildomat/jobs/package.sh

diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
old mode 100644
new mode 100755
index 6a76d09b269..7fca618bcae
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -32,7 +32,7 @@ ptime -m cargo run --locked --release --bin omicron-package -- -t switch_variant
 
 utilities=(
   package-manifest.toml
-  mf/sled-agent/config.toml
+  smf/sled-agent/config.toml
   tools/create_virtual_hardware.sh
 )
 

From 8874d34d1b4bd0663851fc24c288d84f224bbf1e Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Mon, 6 Mar 2023 11:28:50 -0500
Subject: [PATCH 12/22] layered fs approach

---
 .github/buildomat/jobs/deploy.sh  | 14 +++++++----
 .github/buildomat/jobs/package.sh | 39 ++++++++++++++++++++++++-------
 2 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index fc1e1b5b540..79cd61126d8 100644
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -110,9 +110,17 @@ pfexec chown build:build /opt/oxide/work
 cd /opt/oxide/work
 
 ptime -m tar xvzf /input/package/work/utilities-package.tar.gz
+
+# Unpack all global zone files into "out".
+# Note that because this is packaged as a layered filesystem, most files
+# exist under "root/".
 mkdir out/
 ptime -m tar xvzf /input/package/work/global-zone-packages.tar.gz -C out
-cp /input/package/work/zones/* out/
+pfexec mv out/root/opt/oxide/sled-agent /opt/oxide/sled-agent
+pfexec mv out/root/opt/oxide/mg-ddm /opt/oxide/mg-ddm
+
+# Move all global zones to their installed location
+pfexec cp /input/package/work/zones/* /opt/oxide
 
 mkdir tests
 for p in /input/build-end-to-end-tests/work/*.gz; do
@@ -153,9 +161,7 @@ sed -e 's/^# address =.*$/address = "192.168.1.199"/' \
 	-e "s/^mac =.*$/mac = \"$(dladm show-phys -m -p -o ADDRESS | head -n 1)\"/" \
 	-i out/sled-agent/pkg/config-rss.toml
 
-pfexec mv out/sled-agent /opt/oxide/
-pfexec mv out/mg-ddm /opt/oxide/
-pfexec svccfg import /opt/oxide/sled-agent/pkg/manifest.xml
+pfexec svccfg import out/root/lib/svc/manifest/site/sled-agent.xml
 
 ./tests/bootstrap
 rm ./tests/bootstrap
diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
index 7fca618bcae..2ebbffb392d 100755
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -26,6 +26,7 @@ rustc --version
 ptime -m ./tools/install_builder_prerequisites.sh -yp
 ptime -m cargo run --locked --release --bin omicron-package -- -t switch_variant=asic package
 
+tarball_src_dir="$(pwd)/out"
 
 # Assemble some utilities into a tarball that can be used by deployment
 # phases of buildomat.
@@ -39,15 +40,37 @@ utilities=(
 ptime -m tar cvzf /work/utilities-package.tar.gz "${utilities[@]}"
 
 # Assemble global zone files in a temporary directory.
-tmp=$(mktemp -d)
-mkdir -p "${tmp}/sled-agent"
-tar -xvzf out/omicron-sled-agent.tar -C "${tmp}/sled-agent"
-mkdir -p "${tmp}/mg-ddm"
-tar -xvzf out/maghemite.tar -C "${tmp}/mg-ddm"
+if ! tmp=$(mktemp -d); then
+  exit 1
+fi
+trap 'cd /; rm -rf "$tmp"' EXIT
+
+# Header file, identifying this is intended to be layered in the global zone.
+# Within the ramdisk, this means that all files under "root/foo" should appear
+# in the global zone as "/foo".
+echo '{"v":"1","t":"layer"}' > "$tmp/oxide.json"
+
+# Extract the sled-agent tarball for re-packaging into the layered GZ archive.
+pkg_dir="$tmp/root/opt/oxide/sled-agent"
+mkdir -p "$pkg_dir"
+cd "$pkg_dir"
+tar -xvfz "$tarball_src_dir/omicron-sled-agent.tar"
+# Ensure that the manifest for the sled agent exists in a location where it may
+# be automatically initialized.
+mkdir -p "$tmp/root/lib/svc/manifest/site/"
+mv pkg/manifest.xml "$tmp/root/lib/svc/manifest/site/sled-agent.xml"
+cd -
+
+# Extract the mg-ddm tarball for re-packaging into the layered GZ archive.
+pkg_dir="$tmp/root/opt/oxide/mg-ddm"
+mkdir -p "$pkg_dir"
+cd "$pkg_dir"
+tar -xvfz "$tarball_src_dir/maghemite.tar"
+cd -
 
-# Load those global zone files into a tarball that's ready to be exported.
 mkdir -p /work
-ptime -m tar cvzf /work/global-zone-packages.tar.gz -C "${tmp}" .
+cd "$tmp" && tar cvfz /work/global-zone-packages.tar.gz oxide.json root
+cd -
 
 # Assemble Zone Images into their respective output locations.
 mkdir -p /work/zones
@@ -63,4 +86,4 @@ zones=(
 	out/propolis-server.tar.gz
 	out/switch-asic.tar.gz
 )
-mv "${zones[@]}" /work/zones/
+cp "${zones[@]}" /work/zones/

From 0055c1647e5321e4873ef2bbf6426e39c3c9ad63 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Mon, 6 Mar 2023 11:36:01 -0500
Subject: [PATCH 13/22] Specify image type

---
 .github/buildomat/jobs/package.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
index 2ebbffb392d..ec822486bf6 100755
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -24,7 +24,7 @@ rustc --version
 
 # Build
 ptime -m ./tools/install_builder_prerequisites.sh -yp
-ptime -m cargo run --locked --release --bin omicron-package -- -t switch_variant=asic package
+ptime -m cargo run --locked --release --bin omicron-package -- -t 'image_type=standard switch_variant=asic' package
 
 tarball_src_dir="$(pwd)/out"
 

From 73b4cec2d0bc6cef76ae20b64a7e94904fec15be Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Mon, 6 Mar 2023 12:17:50 -0500
Subject: [PATCH 14/22] path to config files

---
 .github/buildomat/jobs/deploy.sh | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index 79cd61126d8..a9b29ab8810 100644
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -116,6 +116,13 @@ ptime -m tar xvzf /input/package/work/utilities-package.tar.gz
 # exist under "root/".
 mkdir out/
 ptime -m tar xvzf /input/package/work/global-zone-packages.tar.gz -C out
+
+# Modify config-rss.toml in the sled-agent to use our system's IP and MAC
+# address for upstream connectivity.
+sed -e 's/^# address =.*$/address = "192.168.1.199"/' \
+	-e "s/^mac =.*$/mac = \"$(dladm show-phys -m -p -o ADDRESS | head -n 1)\"/" \
+	-i out/root/opt/oxide/sled-agent/pkg/config-rss.toml
+
 pfexec mv out/root/opt/oxide/sled-agent /opt/oxide/sled-agent
 pfexec mv out/root/opt/oxide/mg-ddm /opt/oxide/mg-ddm
 
@@ -155,12 +162,6 @@ pfexec svccfg import /var/svc/manifest/site/tcpproxy.xml
 #
 pfexec ipadm create-addr -T static -a 192.168.1.199/24 igb0/sidehatch
 
-# Modify config-rss.toml in the sled-agent to use our system's IP and MAC
-# address for upstream connectivity.
-sed -e 's/^# address =.*$/address = "192.168.1.199"/' \
-	-e "s/^mac =.*$/mac = \"$(dladm show-phys -m -p -o ADDRESS | head -n 1)\"/" \
-	-i out/sled-agent/pkg/config-rss.toml
-
 pfexec svccfg import out/root/lib/svc/manifest/site/sled-agent.xml
 
 ./tests/bootstrap

From 331660454c5e48c14512d32c3ee4f34083873316 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Mon, 6 Mar 2023 12:52:30 -0500
Subject: [PATCH 15/22] Make the gz package generation purely additive for now

---
 .github/buildomat/jobs/deploy.sh  | 44 ++++++++++++++++---------------
 .github/buildomat/jobs/package.sh | 16 ++++++-----
 2 files changed, 33 insertions(+), 27 deletions(-)

diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh
index a9b29ab8810..6217e33d794 100644
--- a/.github/buildomat/jobs/deploy.sh
+++ b/.github/buildomat/jobs/deploy.sh
@@ -109,26 +109,8 @@ pfexec mkdir /opt/oxide/work
 pfexec chown build:build /opt/oxide/work
 cd /opt/oxide/work
 
-ptime -m tar xvzf /input/package/work/utilities-package.tar.gz
-
-# Unpack all global zone files into "out".
-# Note that because this is packaged as a layered filesystem, most files
-# exist under "root/".
-mkdir out/
-ptime -m tar xvzf /input/package/work/global-zone-packages.tar.gz -C out
-
-# Modify config-rss.toml in the sled-agent to use our system's IP and MAC
-# address for upstream connectivity.
-sed -e 's/^# address =.*$/address = "192.168.1.199"/' \
-	-e "s/^mac =.*$/mac = \"$(dladm show-phys -m -p -o ADDRESS | head -n 1)\"/" \
-	-i out/root/opt/oxide/sled-agent/pkg/config-rss.toml
-
-pfexec mv out/root/opt/oxide/sled-agent /opt/oxide/sled-agent
-pfexec mv out/root/opt/oxide/mg-ddm /opt/oxide/mg-ddm
-
-# Move all global zones to their installed location
-pfexec cp /input/package/work/zones/* /opt/oxide
-
+ptime -m tar xvzf /input/package/work/package.tar.gz
+cp /input/package/work/zones/* out/
 mkdir tests
 for p in /input/build-end-to-end-tests/work/*.gz; do
 	ptime -m gunzip < "$p" > "tests/$(basename "${p%.gz}")"
@@ -162,7 +144,27 @@ pfexec svccfg import /var/svc/manifest/site/tcpproxy.xml
 #
 pfexec ipadm create-addr -T static -a 192.168.1.199/24 igb0/sidehatch
 
-pfexec svccfg import out/root/lib/svc/manifest/site/sled-agent.xml
+#
+# Modify config-rss.toml in the sled-agent zone to use our system's IP and MAC
+# address for upstream connectivity.
+#
+tar xf out/omicron-sled-agent.tar pkg/config-rss.toml
+sed -e 's/^# address =.*$/address = "192.168.1.199"/' \
+	-e "s/^mac =.*$/mac = \"$(dladm show-phys -m -p -o ADDRESS | head -n 1)\"/" \
+	-i pkg/config-rss.toml
+tar rf out/omicron-sled-agent.tar pkg/config-rss.toml
+rm -rf pkg
+
+#
+# This OMICRON_NO_UNINSTALL hack here is so that there is no implicit uninstall
+# before the install.  This doesn't work right now because, above, we made
+# /var/oxide a file system so you can't remove it (EBUSY) like a regular
+# directory.  The lab-netdev target is a ramdisk system that is always cleared
+# out between runs, so it has not had any state yet that requires
+# uninstallation.
+#
+OMICRON_NO_UNINSTALL=1 \
+    ptime -m pfexec ./target/release/omicron-package install
 
 ./tests/bootstrap
 rm ./tests/bootstrap
diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
index ec822486bf6..b5eee627053 100755
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -5,7 +5,7 @@
 #: target = "helios-latest"
 #: rust_toolchain = "1.66.1"
 #: output_rules = [
-#:	"=/work/utilities-package.tar.gz",
+#:	"=/work/package.tar.gz",
 #:	"=/work/global-zone-packages.tar.gz",
 #:	"=/work/zones/*.tar.gz",
 #: ]
@@ -25,19 +25,22 @@ rustc --version
 # Build
 ptime -m ./tools/install_builder_prerequisites.sh -yp
 ptime -m cargo run --locked --release --bin omicron-package -- -t 'image_type=standard switch_variant=asic' package
+ptime -m cargo run --locked --release --bin omicron-package -- -t 'image_type=standard switch_variant=stub' package
 
 tarball_src_dir="$(pwd)/out"
 
 # Assemble some utilities into a tarball that can be used by deployment
 # phases of buildomat.
 
-utilities=(
-  package-manifest.toml
-  smf/sled-agent/config.toml
-  tools/create_virtual_hardware.sh
+files=(
+	out/*.tar
+	package-manifest.toml
+	smf/sled-agent/config.toml
+	target/release/omicron-package
+	tools/create_virtual_hardware.sh
 )
 
-ptime -m tar cvzf /work/utilities-package.tar.gz "${utilities[@]}"
+ptime -m tar cvzf /work/package.tar.gz "${files[@]}"
 
 # Assemble global zone files in a temporary directory.
 if ! tmp=$(mktemp -d); then
@@ -85,5 +88,6 @@ zones=(
 	out/oximeter-collector.tar.gz
 	out/propolis-server.tar.gz
 	out/switch-asic.tar.gz
+	out/switch-stub.tar.gz
 )
 cp "${zones[@]}" /work/zones/

From 6474dba95a3dcff397db99da357984195135dae4 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Mon, 6 Mar 2023 16:50:16 -0500
Subject: [PATCH 16/22] Assemble installinator files into an overlay tarball

---
 .github/buildomat/jobs/package.sh | 63 ++++++++++++++++++++++++++-----
 1 file changed, 54 insertions(+), 9 deletions(-)

diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
index b5eee627053..678a9ab8100 100755
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -7,6 +7,7 @@
 #: output_rules = [
 #:	"=/work/package.tar.gz",
 #:	"=/work/global-zone-packages.tar.gz",
+#:	"=/work/trampoline-global-zone-packages.tar.gz",
 #:	"=/work/zones/*.tar.gz",
 #: ]
 #:
@@ -14,6 +15,11 @@
 #: series = "image"
 #: name = "global-zone-packages"
 #: from_output = "/out/global-zone-packages.tar.gz"
+#:
+#: [[publish]]
+#: series = "image"
+#: name = "trampoline-global-zone-packages"
+#: from_output = "/out/trampoline-global-zone-packages.tar.gz"
 
 set -o errexit
 set -o pipefail
@@ -26,6 +32,7 @@ rustc --version
 ptime -m ./tools/install_builder_prerequisites.sh -yp
 ptime -m cargo run --locked --release --bin omicron-package -- -t 'image_type=standard switch_variant=asic' package
 ptime -m cargo run --locked --release --bin omicron-package -- -t 'image_type=standard switch_variant=stub' package
+ptime -m cargo run --locked --release --bin omicron-package -- -t 'image_type=trampoline' package
 
 tarball_src_dir="$(pwd)/out"
 
@@ -42,39 +49,77 @@ files=(
 
 ptime -m tar cvzf /work/package.tar.gz "${files[@]}"
 
-# Assemble global zone files in a temporary directory.
-if ! tmp=$(mktemp -d); then
+#
+# Global Zone files for Host OS
+#
+
+if ! tmp_gz=$(mktemp -d); then
   exit 1
 fi
-trap 'cd /; rm -rf "$tmp"' EXIT
+trap 'cd /; rm -rf "$tmp_gz"' EXIT
 
 # Header file, identifying this is intended to be layered in the global zone.
 # Within the ramdisk, this means that all files under "root/foo" should appear
 # in the global zone as "/foo".
-echo '{"v":"1","t":"layer"}' > "$tmp/oxide.json"
+echo '{"v":"1","t":"layer"}' > "$tmp_gz/oxide.json"
 
 # Extract the sled-agent tarball for re-packaging into the layered GZ archive.
-pkg_dir="$tmp/root/opt/oxide/sled-agent"
+pkg_dir="$tmp_gz/root/opt/oxide/sled-agent"
 mkdir -p "$pkg_dir"
 cd "$pkg_dir"
 tar -xvfz "$tarball_src_dir/omicron-sled-agent.tar"
 # Ensure that the manifest for the sled agent exists in a location where it may
 # be automatically initialized.
-mkdir -p "$tmp/root/lib/svc/manifest/site/"
-mv pkg/manifest.xml "$tmp/root/lib/svc/manifest/site/sled-agent.xml"
+mkdir -p "$tmp_gz/root/lib/svc/manifest/site/"
+mv pkg/manifest.xml "$tmp_gz/root/lib/svc/manifest/site/sled-agent.xml"
+cd -
+# Extract the mg-ddm tarball for re-packaging into the layered GZ archive.
+pkg_dir="$tmp_gz/root/opt/oxide/mg-ddm"
+mkdir -p "$pkg_dir"
+cd "$pkg_dir"
+tar -xvfz "$tarball_src_dir/maghemite.tar"
+cd -
+
+mkdir -p /work
+cd "$tmp_gz" && tar cvfz /work/global-zone-packages.tar.gz oxide.json root
 cd -
 
+#
+# Global Zone files for for Trampoline image
+#
+
+if ! tmp_trampoline=$(mktemp -d); then
+  exit 1
+fi
+trap 'cd /; rm -rf "$tmp_trampoline"' EXIT
+
+echo '{"v":"1","t":"layer"}' > "$tmp_trampoline/oxide.json"
+
+# Extract the installinator tarball for re-packaging into the layered GZ archive.
+pkg_dir="$tmp_trampoline/root/opt/oxide/installinator"
+mkdir -p "$pkg_dir"
+cd "$pkg_dir"
+tar -xvfz "$tarball_src_dir/installinator.tar"
+# Ensure that the manifest for the installinator exists in a location where it may
+# be automatically initialized.
+mkdir -p "$tmp_trampoline/root/lib/svc/manifest/site/"
+mv pkg/manifest.xml "$tmp_trampoline/root/lib/svc/manifest/site/installinator.xml"
+cd -
 # Extract the mg-ddm tarball for re-packaging into the layered GZ archive.
-pkg_dir="$tmp/root/opt/oxide/mg-ddm"
+pkg_dir="$tmp_trampoline/root/opt/oxide/mg-ddm"
 mkdir -p "$pkg_dir"
 cd "$pkg_dir"
 tar -xvfz "$tarball_src_dir/maghemite.tar"
 cd -
 
 mkdir -p /work
-cd "$tmp" && tar cvfz /work/global-zone-packages.tar.gz oxide.json root
+cd "$tmp_trampoline" && tar cvfz /work/trampoline-global-zone-packages.tar.gz oxide.json root
 cd -
 
+#
+# Non-Global Zones
+#
+
 # Assemble Zone Images into their respective output locations.
 mkdir -p /work/zones
 zones=(

From 143b82647f520434cff321a4375676b739e7734a Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Mon, 6 Mar 2023 22:15:51 -0500
Subject: [PATCH 17/22] [buildomat] Job to create Helios trampoline image

---
 .github/buildomat/jobs/recovery-image.sh | 89 ++++++++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 .github/buildomat/jobs/recovery-image.sh

diff --git a/.github/buildomat/jobs/recovery-image.sh b/.github/buildomat/jobs/recovery-image.sh
new file mode 100644
index 00000000000..c7007fa5a0a
--- /dev/null
+++ b/.github/buildomat/jobs/recovery-image.sh
@@ -0,0 +1,89 @@
+#!/bin/bash
+#:
+#: name = "helios / build recovery OS image"
+#: variety = "basic"
+#: target = "helios-latest"
+#: rust_toolchain = "1.66.1"
+#: output_rules = [
+#:	"=/work/helios/image/output/zfs.img",
+#:	"=/work/helios/image/output/rom",
+#: ]
+#: skip_clone = true
+#: access_repos = [
+#:	"oxidecomputer/amd-apcb",
+#:	"oxidecomputer/amd-efs",
+#:	"oxidecomputer/amd-firmware",
+#:	"oxidecomputer/amd-flash",
+#:	"oxidecomputer/amd-host-image-builder",
+#:	"oxidecomputer/boot-image-tools",
+#:	"oxidecomputer/chelsio-t6-roms",
+#:	"oxidecomputer/helios",
+#:	"oxidecomputer/helios-omnios-build",
+#:	"oxidecomputer/helios-omnios-extra",
+#:	"oxidecomputer/nanobl-rs",
+#: ]
+#:
+#: [dependencies.package]
+#: job = "helios / package"
+#:
+
+set -o errexit
+set -o pipefail
+set -o xtrace
+
+cargo --version
+rustc --version
+
+#
+# The token authentication mechanism that affords us access to other private
+# repositories requires that we use HTTPS URLs for GitHub, rather than SSH.
+#
+override_urls=(
+    'git://github.com/'
+    'git@github.com:'
+    'ssh://github.com/'
+    'ssh://git@github.com/'
+)
+for (( i = 0; i < ${#override_urls[@]}; i++ )); do
+	git config --add --global url.https://github.com/.insteadOf \
+	    "${override_urls[$i]}"
+done
+
+#
+# Require that cargo use the git CLI instead of the built-in support.  This
+# achieves two things: first, SSH URLs should be transformed on fetch without
+# requiring Cargo.toml rewriting, which is especially difficult in transitive
+# dependencies; second, Cargo does not seem willing on its own to look in
+# ~/.netrc and find the temporary token that buildomat generates for our job,
+# so we must use git which uses curl.
+#
+export CARGO_NET_GIT_FETCH_WITH_CLI=true
+
+pfexec mkdir -p /work
+cd /work
+
+# /work/gz: Global Zone artifacts to be placed in the Helios image.
+mkdir gz && cd gz
+ptime -m tar xvzf /input/package/work/trampoline-global-zone-packages.tar.gz
+cd -
+
+# TODO: Consider importing zones here too?
+
+# Checkout helios at a pinned commit
+git clone https://github.com/oxidecomputer/helios.git
+cd helios
+git checkout ac8a7e7ef9e9b5ef27334bc8016f5d123f852449
+
+# Create the "./helios-build" command, which lets us build images
+gmake setup
+
+# Commands that "./helios-build" would ask us to run (either explicitly
+# or implicitly, to avoid an error).
+pfexec pkg install /system/zones/brand/omicron1/tools
+pfexec zfs create -p rpool/images/build
+
+./helios-build experiment-image \
+	-p helios-netdev=https://pkg.oxide.computer/helios-netdev \
+	-F optever=0.21 \
+	-P /work/gz/root \
+	-B

From c1322085907c06932aef7021dd9e93e56ba85f18 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Tue, 7 Mar 2023 09:52:58 -0500
Subject: [PATCH 18/22] Tweak flags for recovery

---
 .github/buildomat/jobs/recovery-image.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/buildomat/jobs/recovery-image.sh b/.github/buildomat/jobs/recovery-image.sh
index c7007fa5a0a..e7f100adb99 100644
--- a/.github/buildomat/jobs/recovery-image.sh
+++ b/.github/buildomat/jobs/recovery-image.sh
@@ -86,4 +86,4 @@ pfexec zfs create -p rpool/images/build
 	-p helios-netdev=https://pkg.oxide.computer/helios-netdev \
 	-F optever=0.21 \
 	-P /work/gz/root \
-	-B
+	-R

From 296e331196b0698bd4e522f57d21c78fedbe58fb Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Tue, 7 Mar 2023 10:26:02 -0500
Subject: [PATCH 19/22] Publish paths under '/work'

---
 .github/buildomat/jobs/package.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
index 678a9ab8100..2f7c561af5d 100755
--- a/.github/buildomat/jobs/package.sh
+++ b/.github/buildomat/jobs/package.sh
@@ -14,12 +14,12 @@
 #: [[publish]]
 #: series = "image"
 #: name = "global-zone-packages"
-#: from_output = "/out/global-zone-packages.tar.gz"
+#: from_output = "/work/global-zone-packages.tar.gz"
 #:
 #: [[publish]]
 #: series = "image"
 #: name = "trampoline-global-zone-packages"
-#: from_output = "/out/trampoline-global-zone-packages.tar.gz"
+#: from_output = "/work/trampoline-global-zone-packages.tar.gz"
 
 set -o errexit
 set -o pipefail

From 1c951f1762178e576b46d7a17d00c5cc0502a603 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Thu, 9 Mar 2023 23:44:27 -0500
Subject: [PATCH 20/22] source, review

---
 .github/buildomat/jobs/recovery-image.sh | 7 ++++---
 tools/helios_version                     | 1 +
 2 files changed, 5 insertions(+), 3 deletions(-)
 create mode 100644 tools/helios_version

diff --git a/.github/buildomat/jobs/recovery-image.sh b/.github/buildomat/jobs/recovery-image.sh
index e7f100adb99..518a80fa937 100644
--- a/.github/buildomat/jobs/recovery-image.sh
+++ b/.github/buildomat/jobs/recovery-image.sh
@@ -34,6 +34,8 @@ set -o xtrace
 cargo --version
 rustc --version
 
+source "./tools/helios_version"
+
 #
 # The token authentication mechanism that affords us access to other private
 # repositories requires that we use HTTPS URLs for GitHub, rather than SSH.
@@ -67,12 +69,11 @@ mkdir gz && cd gz
 ptime -m tar xvzf /input/package/work/trampoline-global-zone-packages.tar.gz
 cd -
 
-# TODO: Consider importing zones here too?
-
 # Checkout helios at a pinned commit
 git clone https://github.com/oxidecomputer/helios.git
 cd helios
-git checkout ac8a7e7ef9e9b5ef27334bc8016f5d123f852449
+
+git checkout "$COMMIT"
 
 # Create the "./helios-build" command, which lets us build images
 gmake setup
diff --git a/tools/helios_version b/tools/helios_version
new file mode 100644
index 00000000000..1ee72f349ac
--- /dev/null
+++ b/tools/helios_version
@@ -0,0 +1 @@
+COMMIT=49d501d2f37060e29a84a50e9026860315975794

From 163b8662b1931b55cd56f99e74ff8444ccd908be Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 10 Mar 2023 00:36:05 -0500
Subject: [PATCH 21/22] abs

---
 .github/buildomat/jobs/recovery-image.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/buildomat/jobs/recovery-image.sh b/.github/buildomat/jobs/recovery-image.sh
index 518a80fa937..eed9021ed2d 100644
--- a/.github/buildomat/jobs/recovery-image.sh
+++ b/.github/buildomat/jobs/recovery-image.sh
@@ -34,7 +34,7 @@ set -o xtrace
 cargo --version
 rustc --version
 
-source "./tools/helios_version"
+source "/work/oxidecomputer/omicron/tools/helios_version"
 
 #
 # The token authentication mechanism that affords us access to other private

From 0868db2cc717d2d5fd75a9eed838d7f42a1a9f97 Mon Sep 17 00:00:00 2001
From: Sean Klein <sean@oxide.computer>
Date: Fri, 10 Mar 2023 18:47:23 -0500
Subject: [PATCH 22/22] merge w/host changes

---
 .github/buildomat/jobs/host-image.sh     | 1 -
 .github/buildomat/jobs/recovery-image.sh | 6 ++++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/buildomat/jobs/host-image.sh b/.github/buildomat/jobs/host-image.sh
index 726ad313f5c..dcd0e05d364 100644
--- a/.github/buildomat/jobs/host-image.sh
+++ b/.github/buildomat/jobs/host-image.sh
@@ -15,7 +15,6 @@
 #:	"oxidecomputer/amd-flash",
 #:	"oxidecomputer/amd-host-image-builder",
 #:	"oxidecomputer/boot-image-tools",
-#:	"oxidecomputer/boot-image-tools",
 #:	"oxidecomputer/chelsio-t6-roms",
 #:	"oxidecomputer/compliance-pilot",
 #:	"oxidecomputer/facade",
diff --git a/.github/buildomat/jobs/recovery-image.sh b/.github/buildomat/jobs/recovery-image.sh
index eed9021ed2d..2e01259d812 100644
--- a/.github/buildomat/jobs/recovery-image.sh
+++ b/.github/buildomat/jobs/recovery-image.sh
@@ -8,7 +8,6 @@
 #:	"=/work/helios/image/output/zfs.img",
 #:	"=/work/helios/image/output/rom",
 #: ]
-#: skip_clone = true
 #: access_repos = [
 #:	"oxidecomputer/amd-apcb",
 #:	"oxidecomputer/amd-efs",
@@ -17,6 +16,8 @@
 #:	"oxidecomputer/amd-host-image-builder",
 #:	"oxidecomputer/boot-image-tools",
 #:	"oxidecomputer/chelsio-t6-roms",
+#:	"oxidecomputer/compliance-pilot",
+#:	"oxidecomputer/facade",
 #:	"oxidecomputer/helios",
 #:	"oxidecomputer/helios-omnios-build",
 #:	"oxidecomputer/helios-omnios-extra",
@@ -34,7 +35,7 @@ set -o xtrace
 cargo --version
 rustc --version
 
-source "/work/oxidecomputer/omicron/tools/helios_version"
+source "$(pwd)/tools/helios_version"
 
 #
 # The token authentication mechanism that affords us access to other private
@@ -45,6 +46,7 @@ override_urls=(
     'git@github.com:'
     'ssh://github.com/'
     'ssh://git@github.com/'
+    'git+ssh://git@github.com/'
 )
 for (( i = 0; i < ${#override_urls[@]}; i++ )); do
 	git config --add --global url.https://github.com/.insteadOf \