Cluster Secrets and Buildkit builds

Gemfile.lock

@@ -40,7 +40,7 @@ GEM
       reline (>= 0.3.8)
     json (2.6.3)
     language_server-protocol (3.17.0.3)
-    minitest (5.19.0)
+    minitest (5.25.4)
     mutex_m (0.1.2)
     parallel (1.23.0)
     parser (3.2.2.4)

bin/docs-preview-annotate

@@ -27,12 +27,13 @@ end
 
 json = JSON.parse(response.body)
 result = json["result"].first
+url = result["aliases"]&.first || result["url"]
 
 plan = <<~PLAN
 #### :writing_hand: rails/docs-preview:
 
-* <a href="#{result["url"]}/api">:link: API</a>
-* <a href="#{result["url"]}/guides">:link: Guides</a>
+* <a href="#{url}/api">:link: API</a>
+* <a href="#{url}/guides">:link: Guides</a>
 PLAN
 
 puts plan

lib/buildkite/config/build_context.rb

@@ -117,6 +117,18 @@ def pull_request
       ([ENV["BUILDKITE_PULL_REQUEST"]] - ["false"]).first
     end
 
+    def compute_type
+      ENV["BUILDKITE_COMPUTE_TYPE"] || "self-hosted"
+    end
+
+    def self_hosted?
+      compute_type == "self-hosted"
+    end
+
+    def hosted?
+      !self_hosted?
+    end
+
     def standard_queues
       [nil, "default", "builder"]
     end
@@ -189,8 +201,20 @@ def min_ruby
         Gem::Version.new($1 || "2.0")
       end
 
+      def registry
+        if hosted?
+          ENV["REGISTRY"]
+        else
+          "973266071021.dkr.ecr.us-east-1.amazonaws.com"
+        end
+      end
+
+      def image_name
+        "#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds"
+      end
+
       def remote_image_base
-        "973266071021.dkr.ecr.us-east-1.amazonaws.com/#{"#{build_queue}-" unless standard_queues.include?(build_queue)}builds"
+        [registry, image_name].join("/")
       end
   end
 end

lib/buildkite/config/docker_build.rb

@@ -30,11 +30,16 @@ def cache_from(build_context)
       end
 
       def build_push(build_context)
-        [
-          build_context.local_branch =~ /:/ ?
-            build_context.image_name_for("pr-#{build_context.pull_request}") :
-            build_context.image_name_for("br-#{build_context.local_branch}"),
-        ]
+        if build_context.hosted?
+          build_context.image_name_for(build_context.build_id, prefix: nil)
+        else
+          [
+            build_context.local_branch =~ /:/ ?
+              build_context.image_name_for("pr-#{build_context.pull_request}") :
+              build_context.image_name_for("br-#{build_context.local_branch}"),
+            build_context.image_name_for(build_context.build_id)
+          ]
+        end
       end
     end
 
@@ -66,23 +71,35 @@ def builder(ruby)
             compressed: ".buildkite.tgz"
           }
 
-          plugin :docker_compose, {
-            build: "base",
-            config: ".buildkite/docker-compose.yml",
-            env: %w[PRE_STEPS RACK],
-            "image-name" => build_context.ruby.image_name_for(build_context.build_id),
-            "cache-from" => cache_from(build_context),
-            push: build_push(build_context),
-            "image-repository" => build_context.image_base,
-          }
+          if build_context.hosted?
+            command <<~COMMAND.squish
+              docker build --push
+                --build-arg RUBY_IMAGE=#{build_context.ruby.ruby_image}
+                --tag #{build_push(build_context)}
+                --file .buildkite/Dockerfile .
+            COMMAND
+          else
+            plugin :docker_compose, {
+              build: "base",
+              config: ".buildkite/docker-compose.yml",
+              env: %w[PRE_STEPS RACK],
+              "cache-from" => cache_from(build_context),
+              push: build_push(build_context),
+            }
+          end
 
-          env({
+          env_opts = {
             BUNDLER: build_context.bundler,
             RUBYGEMS: build_context.rubygems,
-            RUBY_IMAGE: build_context.ruby.ruby_image,
             encrypted_0fb9444d0374_key: nil,
             encrypted_0fb9444d0374_iv: nil
-          })
+          }
+
+          if build_context.self_hosted?
+            env_opts[:RUBY_IMAGE] = build_context.ruby.ruby_image
+          end
+
+          env(env_opts)
 
           timeout_in_minutes 15
 

lib/buildkite/config/rake_command.rb

@@ -33,7 +33,7 @@ def build_env(build_context, pre_steps, env)
         env
       end
 
-      def install_plugins(service = "default", env = nil, dir = ".")
+      def install_plugins(service = "default", env = nil, dir = ".", build_context:)
         plugin :artifacts, {
             download: ".dockerignore"
           }
@@ -49,14 +49,25 @@ def install_plugins(service = "default", env = nil, dir = ".")
           compressed: ".buildkite.tgz"
         }
 
-        plugin :docker_compose, {
+        if build_context.mainline
+          plugin :secrets, {
+            env: "main_env"
+          }
+        end
+
+        compose_opts = {
           "env" => env,
           "run" => service,
-          "pull" => service,
-          "pull-retries" => 3,
           "config" => ".buildkite/docker-compose.yml",
           "shell" => ["runner", *dir],
-        }.compact
+          "tty" => "true",
+        }
+
+        if build_context.self_hosted?
+          compose_opts["cli-version"] = "1"
+        end
+
+        plugin :docker_compose, compose_opts.compact
       end
     end
 
@@ -73,7 +84,7 @@ def bundle(command, label:, env: nil)
           depends_on "docker-image-#{build_context.ruby.image_key}"
           command command
 
-          install_plugins
+          install_plugins(build_context: build_context)
 
           env build_env(build_context, nil, env)
 
@@ -99,7 +110,7 @@ def rake(dir, task: "test", label: nil, service: "default", pre_steps: nil, env:
           depends_on "docker-image-#{build_context.ruby.image_key}"
           command "rake #{task}"
 
-          install_plugins(service,  %w[PRE_STEPS RACK], dir)
+          install_plugins(service,  %w[PRE_STEPS RACK], dir, build_context: build_context)
 
           env build_env(build_context, pre_steps, env)
 

pipelines/buildkite-config/initial.yml

@@ -57,6 +57,7 @@ steps:
           -v "$$PWD":/app:ro -w /app
           -v "$$PWD/tmp":/app/tmp:rw
           -e BUNDLE_FROZEN
+          -e BUILDKITE_BUILD_ID
           ruby:latest
           ./bin/pipeline-annotate
     - command: |
@@ -80,6 +81,7 @@ steps:
           -v "$$PWD/tmp":/app/tmp:rw
           -e RAILS_CI_NIGHTLY
           -e BUNDLE_FROZEN
+          -e BUILDKITE_BUILD_ID
           ruby:latest
           ./bin/pipeline-annotate
 

pipelines/docs-preview/initial.yml

@@ -30,6 +30,10 @@ steps:
 
       ([ -f .buildkite/.dockerignore ] && cp .buildkite/.dockerignore .dockerignore) || true
 
+    cache:
+      paths:
+        - "cache/bundler"
+      name: "docs-preview-initial-bundler-cache"
     plugins:
       - artifacts#v1.9.3:
           upload: ".dockerignore"
@@ -48,6 +52,7 @@ steps:
       PIPELINE_COMMAND: >-
         docker run --rm
         -v "$$PWD":/app:ro -w /app
+        -v "$$PWD/cache/bundler":/usr/local/bundle
         -e CI
         -e BUILDKITE
         -e BUILDKITE_AGENT_META_DATA_QUEUE

pipelines/docs-preview/pipeline.rb

@@ -3,9 +3,11 @@
 Buildkite::Builder.pipeline do
   require "buildkite_config"
   use Buildkite::Config::BuildContext
+  use Buildkite::Config::DockerBuild
 
   plugin :docker, "docker#v5.10.0"
   plugin :artifacts, "artifacts#v1.9.3"
+  plugin :secrets, "cluster-secrets#v1.0.0"
 
   build_context = context.extensions.find(Buildkite::Config::BuildContext)
   build_context.ruby = Buildkite::Config::RubyConfig.new(prefix: "ruby:", version: Gem::Version.new("3.3"))
@@ -22,13 +24,16 @@
     next
   end
 
+  builder build_context.ruby
+
   command do
     label "build", emoji: :rails
+    depends_on "docker-image-#{build_context.ruby.image_key}"
     key "build"
     command "bundle install && bundle exec rake preview_docs"
     timeout_in_minutes 15
     plugin :docker, {
-      image: build_context.image_name_for("br-main", prefix: nil),
+      image: build_context.image_name_for(build_context.build_id, prefix: nil),
       environment: [
         "BUILDKITE_BRANCH",
         "BUILDKITE_BUILD_CREATOR",
@@ -50,6 +55,9 @@
     key "deploy"
     depends_on "build"
     timeout_in_minutes 15
+    plugin :secrets, {
+      env: "docs_preview_env"
+    }
     plugin :docker, {
       environment: [
         "BUILDKITE_BRANCH",
@@ -83,6 +91,9 @@
       download: ".buildkite/bin/docs-preview-annotate",
       compressed: ".buildkite.tgz"
     }
+    plugin :secrets, {
+      env: "docs_preview_env"
+    }
     command "sh -c \"$$ANNOTATE_COMMAND\" | buildkite-agent annotate --style info"
     # CLOUDFLARE_API_TOKEN is used to fetch preview URL from latest deployment
     env "ANNOTATE_COMMAND" => <<~ANNOTATE.gsub(/[[:space:]]+/, " ").strip

pipelines/rails-ci-nightly/initial.yml

@@ -30,6 +30,11 @@ steps:
 
       ([ -f .buildkite/.dockerignore ] && cp .buildkite/.dockerignore .dockerignore) || true
 
+    cache:
+      paths:
+        - "cache/bundler"
+      name: "rails-initial-bundler-cache"
+
     plugins:
       - artifacts#v1.9.3:
           upload: ".dockerignore"
@@ -48,6 +53,7 @@ steps:
       PIPELINE_COMMAND: >-
         docker run --rm
         -v "$$PWD":/app:ro -w /app
+        -v "$$PWD/cache/bundler":/usr/local/bundle
         -e RAILS_CI_NIGHTLY
         -e CI
         -e BUILDKITE

pipelines/rails-ci/initial.yml

@@ -4,6 +4,9 @@
 steps:
   - name: ":pipeline: rails-initial-pipeline"
     command: |
+      echo "Fetching registry details"
+      export REGISTRY="$$(nsc workspace describe -o json -k registry_url)"
+
       PATH=/bin:/usr/bin
       set -e
 
@@ -30,6 +33,7 @@ steps:
       echo "Fetching pull-request metadata:"
       (docker run --rm \
         -v "$$PWD":/app:ro -w /app \
+        -v "$$PWD/cache/bundler":/usr/local/bundle \
         -e GITHUB_PUBLIC_REPO_TOKEN \
         -e BUILDKITE_REPO \
         -e BUILDKITE_PULL_REQUEST \
@@ -40,7 +44,10 @@ steps:
       sh -c "$$PIPELINE_COMMAND"
 
       ([ -f .buildkite/.dockerignore ] && cp .buildkite/.dockerignore .dockerignore) || true
-
+    cache:
+      paths:
+        - "cache/bundler"
+      name: "rails-initial-bundler-cache"
     plugins:
       - artifacts#v1.9.3:
           upload: ".dockerignore"
@@ -58,6 +65,7 @@ steps:
       PIPELINE_COMMAND: >-
         docker run --rm
         -v "$$PWD":/app:ro -w /app
+        -v "$$PWD/cache/bundler":/usr/local/bundle
         -e CI
         -e BUILDKITE
         -e BUILDKITE_AGENT_META_DATA_QUEUE
@@ -72,6 +80,7 @@ steps:
         -e DOCKER_IMAGE
         -e RUN_QUEUE
         -e QUEUE
+        -e REGISTRY
         ruby:latest
         .buildkite/bin/pipeline-generate rails-ci |
         buildkite-agent pipeline upload

pipelines/rails-ci/pipeline.rb

@@ -7,8 +7,9 @@
   use Buildkite::Config::RakeCommand
   use Buildkite::Config::RubyGroup
 
-  plugin :docker_compose, "docker-compose#v4.16.0"
+  plugin :docker_compose, "docker-compose#v5.6.0"
   plugin :artifacts, "artifacts#v1.9.3"
+  plugin :secrets, "cluster-secrets#v1.0.0"
 
   if build_context.nightly?
     build_context.rubies << Buildkite::Config::RubyConfig.master_ruby