dist: build openwrt packages (scionproto#4464)

The method is the following:

* Add the prebuild openwrt SDKs for the desired target platform as an
http_archive dependency.
* Slap a BUILD.bazel file on top of the SDK so bazel can make it do two
things:
* Configure a bazel toolchain around the compiler suite embedded in the
SDK (musl-gcc/musl-libc).
* Make one openwrt package for each SCION component (the pieces are
provided by the SCION main workspace).
* Declare a platform for each supported openwrt target.
* Bind each openwrt toolchain to its matching platform.
* Add rules to produce ipk packages for scion components (they just
reference and copy the packages produced by the openwrt workspace).
* Use multiplatform_filegroup() to transition these packages to each
supported openwrt platform.
* Add a Make target, dist-openwrt, to build the openwrt file group.

That results in the following chain of dependencies for one given
component and the amd64 target:
=> Want openwrt
=> build openwrt_amd64 (among others, if any)
=> build "component.ipk for openwrt_amd64"
=> copy "component.ipk for openwrt_amd64" from
@external/openwrtSDK_x86_64
=> @external/openwrtSDK_x86_64 obtains "component exec for
openwrt_amd64" from @scion, then embed in component.ipk package.
=> @scion builds component with the openwrt_amd64 toolchain

Embellishments:

* Optionally generate versioned file names for installables. That
removes the need for renaming the files after the build.
* Added coremark to our build (and as an openwrt package). The intent is
to use it in benchmarking as a rough cpu power metric.
* Executable compression when packaging for openwrt, as these devices
can be storage constrained and Go executables are huge. Unfortunately
there's not much I can do for the in-ram size yet.

<!-- Reviewable:start -->
- - -
This change is [<img src="https://reviewable.io/review_button.svg"
height="34" align="absmiddle"
alt="Reviewable"/>](https://reviewable.io/reviews/scionproto/scion/4464)
<!-- Reviewable:end -->

Author: jiceatscion

.bazelrc

@@ -14,6 +14,9 @@ build --workspace_status_command=./tools/bazel-build-env
 build --java_runtime_version=remotejdk_11
 # disable legacy_create_init for py_binary, py_test etc. This may eventually become the default.
 build --incompatible_default_to_explicit_init_py
+# Enable resolution of cc toolchain by go toolchain
+build --incompatible_enable_cc_toolchain_resolution
+build --flag_alias=file_name_version=//:file_name_version
 
 # include one of "--define gotags=sqlite_mattn" or "--define gotags=sqlite_modernc"
 # cannot be in common, because query chokes on it.

.buildkite/pipeline.yml

@@ -23,27 +23,32 @@ steps:
         - exit_status: 255 # Forced agent shutdown
     timeout_in_minutes: 10
   - wait
-  - label: "Package :debian:"
+  - label: "Package :debian: :openwrt:"
     command:
-      - make dist-deb
-      - cd deb;
-      - tar -chaf scion-deb-amd64.tar.gz *_amd64.deb
-      - tar -chaf scion-deb-arm64.tar.gz *_arm64.deb
-      - tar -chaf scion-deb-i386.tar.gz *_i386.deb
-      - tar -chaf scion-deb-armel.tar.gz *_armel.deb
+      - version="$(tools/git-version)"
+      - make dist-deb BFLAGS="--file_name_version=${version}"
+      - make dist-openwrt BFLAGS="--file_name_version=${version}"
+      - cd installables;
+      - tar -chaf scion-deb-amd64.tar.gz *_${version}_amd64.deb
+      - tar -chaf scion-deb-arm64.tar.gz *_${version}_arm64.deb
+      - tar -chaf scion-deb-i386.tar.gz *_${version}_i386.deb
+      - tar -chaf scion-deb-armel.tar.gz *_${version}_armel.deb
+      - tar -chaf scion-openwrt-x86_64.tar.gz *_${version}_x86_64.ipk
     artifact_paths:
-      - "deb/*.tar.gz"
+      - "installables/scion-*.tar.gz"
     plugins:
       - scionproto/metahook#v0.3.0:
           post-artifact: |
             cat << EOF | buildkite-agent annotate --style "info" --context "packages"
             #### Packages :debian:
-            - <a href="artifact://deb/scion-deb-amd64.tar.gz">amd64</a>
-            - <a href="artifact://deb/scion-deb-arm64.tar.gz">arm64</a>
-            - <a href="artifact://deb/scion-deb-i386.tar.gz">i386</a>
-            - <a href="artifact://deb/scion-deb-armel.tar.gz">armel</a>
+            - <a href="artifact://installables/scion-deb-amd64.tar.gz">amd64</a>
+            - <a href="artifact://installables/scion-deb-arm64.tar.gz">arm64</a>
+            - <a href="artifact://installables/scion-deb-i386.tar.gz">i386</a>
+            - <a href="artifact://installables/scion-deb-armel.tar.gz">armel</a>
+            #### Packages :openwrt:
+            - <a href="artifact://installables/scion-openwrt-x86_64.tar.gz">x86_64</a>
             EOF
-    key: dist-deb
+    key: dist
     retry: *automatic-retry
   - label: "Unit Tests :bazel:"
     command:

.gitignore

@@ -68,7 +68,7 @@ doc/venv/
 
 # Generated package files
 ##########################
-/deb/
+/installables/
 
 # CTags
 ##########################

BUILD.bazel

@@ -1,4 +1,5 @@
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
+load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
 load("@bazel_gazelle//:def.bzl", "gazelle")
 load("@rules_pkg//:pkg.bzl", "pkg_tar")
 load("@io_bazel_rules_go//go:def.bzl", "nogo")
@@ -156,6 +157,17 @@ flake8_lint_config(
     ],
 )
 
+# Optional version string to produce versioned file names. End deliverables, such as installable
+# packages will have a name derived from that string.
+# The flag is to be used when producing publishable assets (so, typically by the CI build).
+# The rest of the time the assets will have an unversioned name. The version tags embedded
+# in binaries and package manifests are always set. Regardless of this flag.
+string_flag(
+    name = "file_name_version",
+    build_setting_default = "dev",
+    visibility = ["//visibility:public"],
+)
+
 # Add a build flag to enable bundling the management API documentation with the
 # binaries. This can be enabled by passing --//:mgmtapi_bundle_doc=true when
 # invoking bazel

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all build build-dev dist-deb antlr clean docker-images gazelle go.mod licenses mocks protobuf scion-topo test test-integration write_all_source_files
+.PHONY: all build build-dev dist-deb antlr clean docker-images gazelle go.mod licenses mocks protobuf scion-topo test test-integration write_all_source_files git-version
 
 build-dev:
 	rm -f bin/*
@@ -11,20 +11,25 @@ build:
 	bazel build //:scion
 	tar -kxf bazel-bin/scion.tar -C bin
 
+# BFLAGS is optional. It may contain additional command line flags for CI builds. Currently this is:
+# "--file_name_version=$(tools/git-version)" to include the git version in the artifacts names.
 dist-deb:
-	bazel build //dist:deb_all
-	mkdir -p deb; rm -f deb/*;
-	@ # Bazel cannot include the version in the filename, if we want to set it automatically from the git tag.
-	@ # Extract the version from the .deb "control" manifest and expand the "__" in the filename to "_<version>_".
-	@ #   See e.g. https://en.wikipedia.org/wiki/Deb_(file_format)#Control_archive
-	@for f in `bazel cquery //dist:deb_all --output=files 2>/dev/null`; do \
-		if [ -f "$$f" ]; then \
-			bf=`basename $$f`; \
-			v="$$(ar p $$f control.tar.gz | tar -xz --to-stdout ./control | sed -n 's/Version: //p')"; \
-			bfv=$${bf%%__*}_$${v}_$${bf#*__}; \
-			cp -v "$$f" deb/$$bfv; \
-		fi \
-	done
+	bazel build //dist:deb_all $(BFLAGS)
+	@ # These artefacts have unique names but varied locations. Link them somewhere convenient.
+	@ mkdir -p installables
+	@ cd installables ; ln -sfv ../bazel-out/*/bin/dist/*.deb .
+
+dist-openwrt:
+	bazel build //dist:openwrt_all $(BFLAGS)
+	@ # These artefacts have unique names but varied locations. Link them somewhere convenient.
+	@ mkdir -p installables
+	@ cd installables ; ln -sfv ../bazel-out/*/bin/dist/*.ipk .
+
+dist-openwrt-testing:
+	bazel build //dist:openwrt_testing_all $(BFLAGS)
+	@ # These artefacts have unique names but varied locations. Link them somewhere convenient.
+	@ mkdir -p installables
+	@ cd installables ; ln -sfv ../bazel-out/*/bin/dist/*.ipk .
 
 # all: performs the code-generation steps and then builds; the generated code
 # is git controlled, and therefore this is only necessary when changing the
@@ -41,6 +46,7 @@ clean:
 scrub:
 	bazel clean --expunge
 	rm -f bin/*
+	rm -f installables/*
 
 test:
 	bazel test --config=unit_all

WORKSPACE

@@ -299,3 +299,18 @@ npm_translate_lock(
 load("@npm//:repositories.bzl", "npm_repositories")
 
 npm_repositories()
+
+# Support cross building and packaging for openwrt_amd64 via the openwrt SDK
+http_archive(
+    name = "openwrt_x86_64_SDK",
+    build_file = "@//dist/openwrt:BUILD.external.bazel",
+    patch_args = ["-p1"],
+    patches = ["@//dist/openwrt:endian_h.patch"],
+    sha256 = "df9cbce6054e6bd46fcf28e2ddd53c728ceef6cb27d1d7fc54a228f272c945b0",
+    strip_prefix = "openwrt-sdk-23.05.2-x86-64_gcc-12.3.0_musl.Linux-x86_64",
+    urls = ["https://downloads.openwrt.org/releases/23.05.2/targets/x86/64/openwrt-sdk-23.05.2-x86-64_gcc-12.3.0_musl.Linux-x86_64.tar.xz"],
+)
+
+register_toolchains(
+    "//dist/openwrt:x86_64_openwrt_toolchain",
+)

dist/BUILD.bazel

@@ -1,11 +1,42 @@
 load(":package.bzl", "scion_pkg_deb")
+load(":package.bzl", "scion_pkg_ipk")
 load(":platform.bzl", "multiplatform_filegroup")
 load(":git_version.bzl", "git_version")
 
+DEB_PLATFORMS = [
+    "@io_bazel_rules_go//go/toolchain:linux_amd64",
+    "@io_bazel_rules_go//go/toolchain:linux_arm64",
+    "@io_bazel_rules_go//go/toolchain:linux_386",
+    "@io_bazel_rules_go//go/toolchain:linux_arm",
+]
+
+# TODO([email protected]):
+# For now only a single openwrt platform can be in this list. If we allow several, they get
+# built in parallel, which breaks on non-reentrant openwrt makefiles. For a single platform
+# things work because we force the packages to be build serialy by making them depend on
+# each other.
+OPENWRT_PLATFORMS = [
+    "@//dist/openwrt:openwrt_amd64",
+]
+
+# Some files that the openwrt build consumes directly.
+exports_files(
+    [
+        "conffiles/gateway.toml",
+        "conffiles/gateway.json",
+        "conffiles/daemon.toml",
+        "conffiles/dispatcher.toml",
+
+        # Directory itself. Indicates the common root of config files.
+        "conffiles",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 git_version(
     name = "git_version",
     tags = ["manual"],
-    visibility = ["//visibility:private"],
+    visibility = ["@openwrt_x86_64_SDK//:__subpackages__"],
 )
 
 scion_pkg_deb(
@@ -57,7 +88,7 @@ scion_pkg_deb(
 
 scion_pkg_deb(
     name = "daemon_deb",
-    configs = ["conffiles/sciond.toml"],
+    configs = ["conffiles/daemon.toml"],
     depends = [
         "adduser",
     ],
@@ -74,8 +105,8 @@ scion_pkg_deb(
 scion_pkg_deb(
     name = "gateway_deb",
     configs = [
-        "conffiles/sig.json",
-        "conffiles/sig.toml",
+        "conffiles/gateway.json",
+        "conffiles/gateway.toml",
     ],
     depends = [
         "adduser",
@@ -117,5 +148,72 @@ multiplatform_filegroup(
         "router_deb",
         "tools_deb",
     ],
+    target_platforms = DEB_PLATFORMS,
+    visibility = ["//dist:__subpackages__"],
+)
+
+scion_pkg_ipk(
+    name = "persistdbs_ipk",
+)
+
+scion_pkg_ipk(
+    name = "router_ipk",
+)
+
+scion_pkg_ipk(
+    name = "gateway_ipk",
+)
+
+scion_pkg_ipk(
+    name = "control_ipk",
+)
+
+scion_pkg_ipk(
+    name = "dispatcher_ipk",
+)
+
+scion_pkg_ipk(
+    name = "daemon_ipk",
+)
+
+scion_pkg_ipk(
+    name = "tools_ipk",
+)
+
+scion_pkg_ipk(
+    name = "coremark_ipk",
+)
+
+scion_pkg_ipk(
+    name = "testconfig_ipk",
+)
+
+# multiplatform_filegroup creates two targets: <name> and <name>_all.
+# <name> means "local platform".
+# So "build //dist/openwrt" means "make me ipk packages for the local platform only. It wont work,
+# unles you happen to be building on an openwrt system (really?).
+# What you really want is "build //dist/openwrt_all".
+multiplatform_filegroup(
+    name = "openwrt",
+    srcs = [
+        "control_ipk",
+        "daemon_ipk",
+        "dispatcher_ipk",
+        "gateway_ipk",
+        "persistdbs_ipk",
+        "router_ipk",
+        "tools_ipk",
+    ],
+    target_platforms = OPENWRT_PLATFORMS,
+    visibility = ["//dist:__subpackages__"],
+)
+
+multiplatform_filegroup(
+    name = "openwrt_testing",
+    srcs = [
+        "coremark_ipk",
+        "testconfig_ipk",
+    ],
+    target_platforms = OPENWRT_PLATFORMS,
     visibility = ["//dist:__subpackages__"],
 )

dist/conffiles/sciond.toml renamed to dist/conffiles/daemon.toml

@@ -1,5 +1,5 @@
 [gateway]
-traffic_policy_file = "/etc/scion/sig.json"
+traffic_policy_file = "/etc/scion/gateway.json"
 
 [tunnel]
 name = "sig"

dist/conffiles/sig.json renamed to dist/conffiles/gateway.json

@@ -2,16 +2,16 @@ def _git_version_impl(ctx):
     ctx.actions.run_shell(
         outputs = [ctx.outputs.outfile],
         inputs = [ctx.info_file],
-        command = r"sed -n 's/STABLE_GIT_VERSION\s*v\?//p' " + ctx.info_file.path + " > " + ctx.outputs.outfile.path,
+        command = r"sed -n 's/STABLE_GIT_VERSION\s*//p' " + ctx.info_file.path + " > " + ctx.outputs.outfile.path,
     )
 
 git_version = rule(
     doc = """
     Extracts the STABLE_GIT_VERSION from the workspace_status_command output.
     See also .bazelrc and tools/bazel-build-env.
 
-    The output of this rule is a file containing the version only.
-    The leading "v" from the git tag is removed.
+    The output of this rule is a file containing the version only. The leading "v" from the git tag
+    is removed by tools/git-version so workspace_status_command never even sees it.
     """,
     implementation = _git_version_impl,
     outputs = {

dist/conffiles/sig.toml renamed to dist/conffiles/gateway.toml

@@ -0,0 +1,72 @@
+# This BUILD file only takes care of hooking the assets derived
+# from the openwrt SDK into the SCION bazel build.
+#
+# These assets are:
+# * a cross-compilation toolchain that targets openwrt_<arch> (i.e. links with musl-libc).
+# * openwrt (ipk) packaging of scion binaries built with the above.
+#
+# Turning these assets into bazel reeferencable dependencies happens in the context of the external
+# src tree openwrt_<target>_SDK, which receives its own buildfile (BUILD.external.bazel, installed there by
+# the http_archive target "openwrt_<target>_SDK" in //WORKSPACE).
+
+# Some files that our openwrt build needs.
+exports_files(
+    [
+        # Gear to drive the make-based openwrt package build process.
+        "BUILD.external.bazel",
+        "endian_h.patch",
+        "package_makefile.tpl",
+
+        # init.d start/stop files.
+        "initds/router",
+        "initds/gateway",
+        "initds/control",
+        "initds/daemon",
+        "initds/dispatcher",
+        "initds/persistdbs",
+
+        # Directory itself. Indicates the common root of config files.
+        "test_configs",
+
+        # Minimal configuration for testing.
+        "test_configs/topology.json",
+        "test_configs/keys/master0.key",
+        "test_configs/keys/master1.key",
+        "test_configs/router.toml",
+        "test_configs/control.toml",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+# Plumb the musl toolchain of the openwrt SDK as a toolchain we can use here to build SCION
+# components.
+constraint_value(
+    name = "musl_libc",
+    constraint_setting = "@bazel_tools//tools/cpp:cc_compiler",
+    visibility = ["//visibility:public"],
+)
+
+toolchain(
+    name = "x86_64_openwrt_toolchain",
+    exec_compatible_with = [
+        "@platforms//cpu:x86_64",
+        "@platforms//os:linux",
+    ],
+    target_compatible_with = [
+        "@platforms//cpu:x86_64",
+        "@platforms//os:linux",
+        ":musl_libc",
+    ],
+    toolchain = "@openwrt_x86_64_SDK//:x86_64_musl",
+    toolchain_type = "@bazel_tools//tools/cpp:toolchain_type",
+)
+
+platform(
+    name = "openwrt_amd64",
+    constraint_values = [
+        "@platforms//cpu:x86_64",
+        "@platforms//os:linux",
+        ":musl_libc",
+    ],
+    visibility = ["//dist:__subpackages__"],
+)