From a20ae4e135376a5712e48f722b5460bde24c3458 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Feb 2025 18:45:17 +0100 Subject: [PATCH 01/23] cache(gha): don't require url attr if url_v2 is set Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- cache/remotecache/gha/gha.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cache/remotecache/gha/gha.go b/cache/remotecache/gha/gha.go index 59f0c21ee6a8..a7af088e752a 100644 --- a/cache/remotecache/gha/gha.go +++ b/cache/remotecache/gha/gha.go @@ -63,10 +63,6 @@ func getConfig(attrs map[string]string) (*Config, error) { if !ok { scope = "buildkit" } - url, ok := attrs[attrURL] - if !ok { - return nil, errors.Errorf("url not set for github actions cache") - } token, ok := attrs[attrToken] if !ok { return nil, errors.Errorf("token not set for github actions cache") @@ -80,12 +76,19 @@ func getConfig(attrs map[string]string) (*Config, error) { } apiVersionInt = int(i) } + var url string if apiVersionInt != 1 { if v, ok := attrs[attrURLV2]; ok { url = v apiVersionInt = 2 } } + if v, ok := attrs[attrURL]; ok && url == "" { + url = v + } + if url == "" { + return nil, errors.Errorf("url not set for github actions cache") + } // best effort on old clients if apiVersionInt == 0 { if strings.Contains(url, "results-receiver.actions.githubusercontent.com") { From 0688013bf26a2c076b0d88e99532458dced8025d Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 13 Feb 2025 16:07:41 -0800 Subject: [PATCH 02/23] add autoallow and entitlements support to CDI devices Devices can be marked as "automatically allowed" by TOML config or by the CDI spec of specific file via annotation. Device that is is not "automatically allowed" needs to be allowed by the build request by passing entitlement. For example a Dockerfile may not use a device without use invoking the build permitting it. --allow device grants access to any device. --allow device=kind|name grants access to specific device. --allow device=kind|name,alias=kind|name allows mapping kind to a specific device or one device to another. Alias is the name requested by the build and device is the actual device that is being enabled. Signed-off-by: Tonis Tiigi --- client/build_test.go | 14 +-- client/client_test.go | 35 +++--- client/solve.go | 14 +-- cmd/buildctl/build.go | 5 +- cmd/buildctl/build/allow.go | 12 +- cmd/buildctl/debug/workers.go | 2 +- cmd/buildkitd/config/config.go | 5 +- cmd/buildkitd/main.go | 18 ++- cmd/buildkitd/main_containerd_worker.go | 5 +- cmd/buildkitd/main_oci_worker.go | 2 +- control/control.go | 2 +- .../dockerfile/dockerfile_rundevice_test.go | 4 +- .../dockerfile/dockerfile_runnetwork_test.go | 4 +- .../dockerfile/dockerfile_runsecurity_test.go | 6 +- solver/llbsolver/bridge.go | 2 +- solver/llbsolver/cdidevices/manager.go | 80 ++++++++++--- solver/llbsolver/solver.go | 13 ++- solver/llbsolver/vertex.go | 72 +++++++++++- util/entitlements/entitlements.go | 109 ++++++++++++++++-- worker/runc/runc.go | 7 +- 20 files changed, 316 insertions(+), 95 deletions(-) diff --git a/client/build_test.go b/client/build_test.go index 10e746109432..c37d81a83429 100644 --- a/client/build_test.go +++ b/client/build_test.go @@ -1838,7 +1838,7 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox command := []string{"sh", "-c", `cat /proc/self/status | grep CapEff | cut -f 2`} mode := llb.SecurityModeSandbox - var allowedEntitlements []entitlements.Entitlement + var allowedEntitlements []string var assertCaps func(caps uint64) secMode := sb.Value("secmode") if secMode == securitySandbox { @@ -1850,7 +1850,7 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox */ require.Equal(t, uint64(0xa80425fb), caps) } - allowedEntitlements = []entitlements.Entitlement{} + allowedEntitlements = []string{} if expectFail { return } @@ -1869,9 +1869,9 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox require.Equal(t, uint64(0x3fffffffff), caps&0x3fffffffff) } mode = llb.SecurityModeInsecure - allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure} + allowedEntitlements = []string{entitlements.EntitlementSecurityInsecure.String()} if expectFail { - allowedEntitlements = []entitlements.Entitlement{} + allowedEntitlements = []string{} } } @@ -2046,13 +2046,13 @@ func testClientGatewayContainerHostNetworking(t *testing.T, sb integration.Sandb ctx := sb.Context() product := "buildkit_test" - var allowedEntitlements []entitlements.Entitlement + var allowedEntitlements []string netMode := pb.NetMode_UNSET if sb.Value("netmode") == hostNetwork { netMode = pb.NetMode_HOST - allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementNetworkHost} + allowedEntitlements = []string{entitlements.EntitlementNetworkHost.String()} if expectFail { - allowedEntitlements = []entitlements.Entitlement{} + allowedEntitlements = []string{} } } c, err := New(sb.Context(), sb.Address()) diff --git a/client/client_test.go b/client/client_test.go index a8bfa093a9ea..5b79d680e230 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -400,9 +400,9 @@ func testHostNetworking(t *testing.T, sb integration.Sandbox) { t.SkipNow() } netMode := sb.Value("netmode") - var allowedEntitlements []entitlements.Entitlement + var allowedEntitlements []string if netMode == hostNetwork { - allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementNetworkHost} + allowedEntitlements = []string{entitlements.EntitlementNetworkHost.String()} } c, err := New(sb.Context(), sb.Address()) require.NoError(t, err) @@ -1063,7 +1063,7 @@ func testSecurityMode(t *testing.T, sb integration.Sandbox) { workers.CheckFeatureCompat(t, sb, workers.FeatureSecurityMode) command := `sh -c 'cat /proc/self/status | grep CapEff | cut -f 2 > /out'` mode := llb.SecurityModeSandbox - var allowedEntitlements []entitlements.Entitlement + var allowedEntitlements []string var assertCaps func(caps uint64) secMode := sb.Value("secmode") if secMode == securitySandbox { @@ -1075,7 +1075,7 @@ func testSecurityMode(t *testing.T, sb integration.Sandbox) { */ require.Equal(t, uint64(0xa80425fb), caps) } - allowedEntitlements = []entitlements.Entitlement{} + allowedEntitlements = []string{} } else { assertCaps = func(caps uint64) { /* @@ -1091,7 +1091,7 @@ func testSecurityMode(t *testing.T, sb integration.Sandbox) { require.Equal(t, uint64(0x3fffffffff), caps&0x3fffffffff) } mode = llb.SecurityModeInsecure - allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure} + allowedEntitlements = []string{entitlements.EntitlementSecurityInsecure.String()} } c, err := New(sb.Context(), sb.Address()) @@ -1138,13 +1138,13 @@ func testSecurityModeSysfs(t *testing.T, sb integration.Sandbox) { } mode := llb.SecurityModeSandbox - var allowedEntitlements []entitlements.Entitlement + var allowedEntitlements []string secMode := sb.Value("secmode") if secMode == securitySandbox { - allowedEntitlements = []entitlements.Entitlement{} + allowedEntitlements = []string{} } else { mode = llb.SecurityModeInsecure - allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure} + allowedEntitlements = []string{entitlements.EntitlementSecurityInsecure.String()} } c, err := New(sb.Context(), sb.Address()) @@ -1191,7 +1191,7 @@ func testSecurityModeErrors(t *testing.T, sb integration.Sandbox) { require.NoError(t, err) _, err = c.Solve(sb.Context(), def, SolveOpt{ - AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}, + AllowedEntitlements: []string{entitlements.EntitlementSecurityInsecure.String()}, }, nil) require.Error(t, err) require.Contains(t, err.Error(), "security.insecure is not allowed") @@ -11054,22 +11054,26 @@ func testCDI(t *testing.T, sb integration.Sandbox) { defer c.Close() require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(` -cdiVersion: "0.3.0" +cdiVersion: "0.6.0" kind: "vendor1.com/device" devices: - name: foo containerEdits: env: - FOO=injected +annotations: + org.mobyproject.buildkit.device.autoallow: true `), 0600)) require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor2-device.yaml"), []byte(` -cdiVersion: "0.3.0" +cdiVersion: "0.6.0" kind: "vendor2.com/device" devices: - name: bar containerEdits: env: - BAR=injected +annotations: + org.mobyproject.buildkit.device.autoallow: true `), 0600)) busybox := llb.Image("busybox:latest") @@ -11119,7 +11123,7 @@ func testCDIFirst(t *testing.T, sb integration.Sandbox) { defer c.Close() require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(` -cdiVersion: "0.3.0" +cdiVersion: "0.6.0" kind: "vendor1.com/device" devices: - name: foo @@ -11138,6 +11142,8 @@ devices: containerEdits: env: - QUX=injected +annotations: + org.mobyproject.buildkit.device.autoallow: true `), 0600)) busybox := llb.Image("busybox:latest") @@ -11184,7 +11190,7 @@ func testCDIWildcard(t *testing.T, sb integration.Sandbox) { defer c.Close() require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(` -cdiVersion: "0.3.0" +cdiVersion: "0.6.0" kind: "vendor1.com/device" devices: - name: foo @@ -11195,6 +11201,8 @@ devices: containerEdits: env: - BAR=injected +annotations: + org.mobyproject.buildkit.device.autoallow: true `), 0600)) busybox := llb.Image("busybox:latest") @@ -11243,6 +11251,7 @@ cdiVersion: "0.6.0" kind: "vendor1.com/device" annotations: foo.bar.baz: FOO + org.mobyproject.buildkit.device.autoallow: true devices: - name: foo annotations: diff --git a/client/solve.go b/client/solve.go index efdf9fa9f105..57ee82d05669 100644 --- a/client/solve.go +++ b/client/solve.go @@ -7,6 +7,7 @@ import ( "io" "maps" "os" + "slices" "strings" "time" @@ -24,7 +25,6 @@ import ( "github.com/moby/buildkit/solver/pb" spb "github.com/moby/buildkit/sourcepolicy/pb" "github.com/moby/buildkit/util/bklog" - "github.com/moby/buildkit/util/entitlements" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/tonistiigi/fsutil" @@ -45,7 +45,7 @@ type SolveOpt struct { CacheExports []CacheOptionsEntry CacheImports []CacheOptionsEntry Session []session.Attachable - AllowedEntitlements []entitlements.Entitlement + AllowedEntitlements []string SharedSession *session.Session // TODO: refactor to better session syncing SessionPreInitialized bool // TODO: refactor to better session syncing Internal bool @@ -277,7 +277,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG FrontendAttrs: frontendAttrs, FrontendInputs: frontendInputs, Cache: &cacheOpt.options, - Entitlements: entitlementsToPB(opt.AllowedEntitlements), + Entitlements: slices.Clone(opt.AllowedEntitlements), Internal: opt.Internal, SourcePolicy: opt.SourcePolicy, }) @@ -553,11 +553,3 @@ func prepareMounts(opt *SolveOpt) (map[string]fsutil.FS, error) { } return mounts, nil } - -func entitlementsToPB(entitlements []entitlements.Entitlement) []string { - clone := make([]string, len(entitlements)) - for i, e := range entitlements { - clone[i] = string(e) - } - return clone -} diff --git a/cmd/buildctl/build.go b/cmd/buildctl/build.go index fd6a4bf0d8b7..bc95f01cf380 100644 --- a/cmd/buildctl/build.go +++ b/cmd/buildctl/build.go @@ -213,8 +213,7 @@ func buildAction(clicontext *cli.Context) error { attachable = append(attachable, secretProvider) } - allowed, err := build.ParseAllow(clicontext.StringSlice("allow")) - if err != nil { + if err := build.ValidateAllow(clicontext.StringSlice("allow")); err != nil { return err } @@ -258,7 +257,7 @@ func buildAction(clicontext *cli.Context) error { CacheExports: cacheExports, CacheImports: cacheImports, Session: attachable, - AllowedEntitlements: allowed, + AllowedEntitlements: clicontext.StringSlice("allow"), SourcePolicy: srcPol, Ref: ref, } diff --git a/cmd/buildctl/build/allow.go b/cmd/buildctl/build/allow.go index fe43e5676dad..70c66b0efb9a 100644 --- a/cmd/buildctl/build/allow.go +++ b/cmd/buildctl/build/allow.go @@ -4,15 +4,13 @@ import ( "github.com/moby/buildkit/util/entitlements" ) -// ParseAllow parses --allow -func ParseAllow(inp []string) ([]entitlements.Entitlement, error) { - ent := make([]entitlements.Entitlement, 0, len(inp)) +// ValidateAllow parses --allow +func ValidateAllow(inp []string) error { for _, v := range inp { - e, err := entitlements.Parse(v) + _, _, err := entitlements.Parse(v) if err != nil { - return nil, err + return err } - ent = append(ent, e) } - return ent, nil + return nil } diff --git a/cmd/buildctl/debug/workers.go b/cmd/buildctl/debug/workers.go index 58fc6ad10d1a..6045d873ebca 100644 --- a/cmd/buildctl/debug/workers.go +++ b/cmd/buildctl/debug/workers.go @@ -94,7 +94,7 @@ func printWorkersVerbose(tw *tabwriter.Writer, winfo []*client.WorkerInfo) { for _, k := range sortedKeys(d.Annotations) { v := d.Annotations[k] - fmt.Fprintf(tw, "\t\t%s:\t%s\n", k, v) + fmt.Fprintf(tw, "\tAnnotations:\t%s:\t%s\n", k, v) } } fmt.Fprint(tw, "\n") diff --git a/cmd/buildkitd/config/config.go b/cmd/buildkitd/config/config.go index 381effcdc9b7..3222406a42da 100644 --- a/cmd/buildkitd/config/config.go +++ b/cmd/buildkitd/config/config.go @@ -77,8 +77,9 @@ type OTELConfig struct { } type CDIConfig struct { - Disabled *bool `toml:"disabled"` - SpecDirs []string `toml:"specDirs"` + Disabled *bool `toml:"disabled"` + SpecDirs []string `toml:"specDirs"` + AutoAllowed []string `toml:"autoAllowed"` } type GCConfig struct { diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index 77d48620f7a8..49e29943708e 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -39,6 +39,7 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/bboltcachestorage" + "github.com/moby/buildkit/solver/llbsolver/cdidevices" "github.com/moby/buildkit/util/apicaps" "github.com/moby/buildkit/util/appcontext" "github.com/moby/buildkit/util/appdefaults" @@ -846,6 +847,11 @@ func newController(ctx context.Context, c *cli.Context, cfg *config.Config) (*co "s3": s3remotecache.ResolveCacheImporterFunc(), "azblob": azblob.ResolveCacheImporterFunc(), } + + if cfg.CDI.Disabled == nil || !*cfg.CDI.Disabled { + cfg.Entitlements = append(cfg.Entitlements, "device") + } + return control.NewController(control.Opt{ SessionManager: sessionManager, WorkerController: wc, @@ -1047,16 +1053,16 @@ func newMeterProvider(ctx context.Context) (*sdkmetric.MeterProvider, error) { } // getCDIManager returns a new CDI registry with disabled auto-refresh. -func getCDIManager(disabled *bool, specDirs []string) (*cdi.Cache, error) { - if disabled != nil && *disabled { +func getCDIManager(cfg config.CDIConfig) (*cdidevices.Manager, error) { + if cfg.Disabled != nil && *cfg.Disabled { return nil, nil } - if len(specDirs) == 0 { - return nil, errors.New("No CDI specification directories specified") + if len(cfg.SpecDirs) == 0 { + return nil, errors.New("no CDI specification directories specified") } cdiCache, err := func() (*cdi.Cache, error) { cdiCache, err := cdi.NewCache( - cdi.WithSpecDirs(specDirs...), + cdi.WithSpecDirs(cfg.SpecDirs...), cdi.WithAutoRefresh(false), ) if err != nil { @@ -1070,5 +1076,5 @@ func getCDIManager(disabled *bool, specDirs []string) (*cdi.Cache, error) { if err != nil { return nil, errors.Wrapf(err, "CDI registry initialization failure") } - return cdiCache, nil + return cdidevices.NewManager(cdiCache, cfg.AutoAllowed), nil } diff --git a/cmd/buildkitd/main_containerd_worker.go b/cmd/buildkitd/main_containerd_worker.go index 13694a48d5bc..38f0844bb505 100644 --- a/cmd/buildkitd/main_containerd_worker.go +++ b/cmd/buildkitd/main_containerd_worker.go @@ -12,7 +12,6 @@ import ( ctd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/defaults" "github.com/moby/buildkit/cmd/buildkitd/config" - "github.com/moby/buildkit/solver/llbsolver/cdidevices" "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/disk" "github.com/moby/buildkit/util/network/cniprovider" @@ -283,7 +282,7 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([ dns := getDNSConfig(common.config.DNS) - cdiManager, err := getCDIManager(common.config.CDI.Disabled, common.config.CDI.SpecDirs) + cdiManager, err := getCDIManager(common.config.CDI) if err != nil { return nil, err } @@ -345,7 +344,7 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([ ParallelismSem: parallelismSem, TraceSocket: common.traceSocket, Runtime: runtime, - CDIManager: cdidevices.NewManager(cdiManager), + CDIManager: cdiManager, } opt, err := containerd.NewWorkerOpt(workerOpts, ctd.WithTimeout(60*time.Second)) diff --git a/cmd/buildkitd/main_oci_worker.go b/cmd/buildkitd/main_oci_worker.go index ee1ed9eb962f..ec2f36e47916 100644 --- a/cmd/buildkitd/main_oci_worker.go +++ b/cmd/buildkitd/main_oci_worker.go @@ -298,7 +298,7 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker dns := getDNSConfig(common.config.DNS) - cdiManager, err := getCDIManager(common.config.CDI.Disabled, common.config.CDI.SpecDirs) + cdiManager, err := getCDIManager(common.config.CDI) if err != nil { return nil, err } diff --git a/control/control.go b/control/control.go index ca50913190b9..a54425168d6e 100644 --- a/control/control.go +++ b/control/control.go @@ -695,7 +695,7 @@ func toPBCDIDevices(manager *cdidevices.Manager) []*apitypes.CDIDevice { for _, dev := range devs { out = append(out, &apitypes.CDIDevice{ Name: dev.Name, - AutoAllow: true, // TODO + AutoAllow: dev.AutoAllow, Annotations: dev.Annotations, OnDemand: dev.OnDemand, }) diff --git a/frontend/dockerfile/dockerfile_rundevice_test.go b/frontend/dockerfile/dockerfile_rundevice_test.go index 4975fb08fbdc..dfc4cdf0035c 100644 --- a/frontend/dockerfile/dockerfile_rundevice_test.go +++ b/frontend/dockerfile/dockerfile_rundevice_test.go @@ -30,13 +30,15 @@ func testDeviceRunEnv(t *testing.T, sb integration.Sandbox) { f := getFrontend(t, sb) require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(` -cdiVersion: "0.3.0" +cdiVersion: "0.6.0" kind: "vendor1.com/device" devices: - name: foo containerEdits: env: - FOO=injected + annotations: + org.mobyproject.buildkit.device.autoallow: true `), 0600)) dockerfile := []byte(` diff --git a/frontend/dockerfile/dockerfile_runnetwork_test.go b/frontend/dockerfile/dockerfile_runnetwork_test.go index a2f360cd3f04..e0ba322d0b0a 100644 --- a/frontend/dockerfile/dockerfile_runnetwork_test.go +++ b/frontend/dockerfile/dockerfile_runnetwork_test.go @@ -132,7 +132,7 @@ RUN --network=host nc 127.0.0.1 %s | grep foo dockerui.DefaultLocalNameDockerfile: dir, dockerui.DefaultLocalNameContext: dir, }, - AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementNetworkHost}, + AllowedEntitlements: []string{entitlements.EntitlementNetworkHost.String()}, }, nil) hostAllowed := sb.Value("network.host") @@ -180,7 +180,7 @@ RUN --network=none ! nc -z 127.0.0.1 %s dockerui.DefaultLocalNameDockerfile: dir, dockerui.DefaultLocalNameContext: dir, }, - AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementNetworkHost}, + AllowedEntitlements: []string{entitlements.EntitlementNetworkHost.String()}, FrontendAttrs: map[string]string{ "force-network-mode": "host", }, diff --git a/frontend/dockerfile/dockerfile_runsecurity_test.go b/frontend/dockerfile/dockerfile_runsecurity_test.go index 2b1cdbe03130..73203910c96a 100644 --- a/frontend/dockerfile/dockerfile_runsecurity_test.go +++ b/frontend/dockerfile/dockerfile_runsecurity_test.go @@ -71,7 +71,7 @@ RUN --security=insecure ls -l /dev && dd if=/dev/zero of=disk.img bs=20M count=1 dockerui.DefaultLocalNameDockerfile: dir, dockerui.DefaultLocalNameContext: dir, }, - AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}, + AllowedEntitlements: []string{entitlements.EntitlementSecurityInsecure.String()}, }, nil) secMode := sb.Value("security.insecure") @@ -109,7 +109,7 @@ RUN [ "$(cat /proc/self/status | grep CapBnd)" == "CapBnd: 00000000a80425fb" ] dockerui.DefaultLocalNameDockerfile: dir, dockerui.DefaultLocalNameContext: dir, }, - AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}, + AllowedEntitlements: []string{entitlements.EntitlementSecurityInsecure.String()}, }, nil) secMode := sb.Value("security.insecure") @@ -173,7 +173,7 @@ RUN [ "$(cat /proc/self/status | grep CapBnd)" == "CapBnd: 00000000a80425fb" ] dockerui.DefaultLocalNameDockerfile: dir, dockerui.DefaultLocalNameContext: dir, }, - AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}, + AllowedEntitlements: []string{entitlements.EntitlementSecurityInsecure.String()}, }, nil) secMode := sb.Value("security.insecure") diff --git a/solver/llbsolver/bridge.go b/solver/llbsolver/bridge.go index 18b20ba0c785..b3466c65066f 100644 --- a/solver/llbsolver/bridge.go +++ b/solver/llbsolver/bridge.go @@ -138,7 +138,7 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp } dpc := &detectPrunedCacheID{} - edge, err := Load(ctx, def, polEngine, dpc.Load, ValidateEntitlements(ent), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps()) + edge, err := Load(ctx, def, polEngine, dpc.Load, ValidateEntitlements(ent, w.CDIManager()), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps()) if err != nil { return nil, errors.Wrap(err, "failed to load LLB") } diff --git a/solver/llbsolver/cdidevices/manager.go b/solver/llbsolver/cdidevices/manager.go index 1996a41c8f7f..2f207d0e5435 100644 --- a/solver/llbsolver/cdidevices/manager.go +++ b/solver/llbsolver/cdidevices/manager.go @@ -2,6 +2,7 @@ package cdidevices import ( "context" + "strconv" "strings" "github.com/moby/buildkit/solver/pb" @@ -13,7 +14,10 @@ import ( "tags.cncf.io/container-device-interface/pkg/parser" ) -const deviceAnnotationClass = "org.mobyproject.buildkit.device.class" +const ( + deviceAnnotationClass = "org.mobyproject.buildkit.device.class" + deviceAutoAllow = "org.mobyproject.buildkit.device.autoallow" +) var installers = map[string]Setup{} @@ -35,17 +39,38 @@ type Device struct { } type Manager struct { - cache *cdi.Cache - locker *locker.Locker + cache *cdi.Cache + locker *locker.Locker + autoAllowed map[string]struct{} } -func NewManager(cache *cdi.Cache) *Manager { +func NewManager(cache *cdi.Cache, autoAllowed []string) *Manager { + m := make(map[string]struct{}) + for _, d := range autoAllowed { + m[d] = struct{}{} + } return &Manager{ - cache: cache, - locker: locker.New(), + cache: cache, + locker: locker.New(), + autoAllowed: m, } } +func (m *Manager) isAutoAllowed(kind, name string, annotations map[string]string) bool { + if _, ok := m.autoAllowed[name]; ok { + return true + } + if _, ok := m.autoAllowed[kind]; ok { + return true + } + if v, ok := annotations[deviceAutoAllow]; ok { + if b, err := strconv.ParseBool(v); err == nil && b { + return true + } + } + return false +} + func (m *Manager) ListDevices() []Device { devs := m.cache.ListDevices() out := make([]Device, 0, len(devs)) @@ -53,10 +78,11 @@ func (m *Manager) ListDevices() []Device { for _, dev := range devs { kind, _, _ := strings.Cut(dev, "=") dd := m.cache.GetDevice(dev) + annotations := deviceAnnotations(dd) out = append(out, Device{ Name: dev, - AutoAllow: true, // TODO - Annotations: deviceAnnotations(dd), + AutoAllow: m.isAutoAllowed(kind, dev, annotations), + Annotations: annotations, }) kinds[kind] = struct{}{} } @@ -69,20 +95,31 @@ func (m *Manager) ListDevices() []Device { continue } out = append(out, Device{ - Name: k, - OnDemand: true, + Name: k, + OnDemand: true, + AutoAllow: true, }) } return out } +func (m *Manager) GetDevice(name string) Device { + kind, _, _ := strings.Cut(name, "=") + annotations := deviceAnnotations(m.cache.GetDevice(name)) + return Device{ + Name: name, + AutoAllow: m.isAutoAllowed(kind, name, annotations), + Annotations: annotations, + } +} + func (m *Manager) Refresh() error { return m.cache.Refresh() } func (m *Manager) InjectDevices(spec *specs.Spec, devs ...*pb.CDIDevice) error { - pdevs, err := m.parseDevices(devs...) + pdevs, err := m.FindDevices(devs...) if err != nil { return err } else if len(pdevs) == 0 { @@ -93,13 +130,17 @@ func (m *Manager) InjectDevices(spec *specs.Spec, devs ...*pb.CDIDevice) error { return err } -func (m *Manager) parseDevices(devs ...*pb.CDIDevice) ([]string, error) { +func (m *Manager) FindDevices(devs ...*pb.CDIDevice) ([]string, error) { var out []string + if len(devs) == 0 { + return out, nil + } + list := m.cache.ListDevices() for _, dev := range devs { if dev == nil { continue } - pdev, err := m.parseDevice(dev) + pdev, err := m.parseDevice(dev, list) if err != nil { return nil, err } else if len(pdev) == 0 { @@ -110,7 +151,7 @@ func (m *Manager) parseDevices(devs ...*pb.CDIDevice) ([]string, error) { return dedupSlice(out), nil } -func (m *Manager) parseDevice(dev *pb.CDIDevice) ([]string, error) { +func (m *Manager) parseDevice(dev *pb.CDIDevice, all []string) ([]string, error) { var out []string kind, name, _ := strings.Cut(dev.Name, "=") @@ -127,7 +168,7 @@ func (m *Manager) parseDevice(dev *pb.CDIDevice) ([]string, error) { switch name { case "": // first device of kind if no name is specified - for _, d := range m.cache.ListDevices() { + for _, d := range all { if strings.HasPrefix(d, kind+"=") { out = append(out, d) break @@ -135,14 +176,14 @@ func (m *Manager) parseDevice(dev *pb.CDIDevice) ([]string, error) { } case "*": // all devices of kind if the name is a wildcard - for _, d := range m.cache.ListDevices() { + for _, d := range all { if strings.HasPrefix(d, kind+"=") { out = append(out, d) } } default: // the specified device - for _, d := range m.cache.ListDevices() { + for _, d := range all { if d == dev.Name { out = append(out, d) break @@ -150,7 +191,7 @@ func (m *Manager) parseDevice(dev *pb.CDIDevice) ([]string, error) { } if len(out) == 0 { // check class annotation if name unknown - for _, d := range m.cache.ListDevices() { + for _, d := range all { if !strings.HasPrefix(d, kind+"=") { continue } @@ -214,6 +255,9 @@ func (m *Manager) OnDemandInstaller(name string) (func(context.Context) error, b return errors.Wrapf(err, "failed to refresh CDI cache") } + // TODO: this needs to be set as annotation to survive reboot + m.autoAllowed[name] = struct{}{} + return nil }, true } diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 15a1f0911372..2ac9070a1fcf 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -1110,19 +1110,26 @@ func supportedEntitlements(ents []string) []entitlements.Entitlement { if e == string(entitlements.EntitlementSecurityInsecure) { out = append(out, entitlements.EntitlementSecurityInsecure) } + if e == string(entitlements.EntitlementDevice) { + out = append(out, entitlements.EntitlementDevice) + } } return out } func loadEntitlements(b solver.Builder) (entitlements.Set, error) { - var ent entitlements.Set = map[entitlements.Entitlement]struct{}{} + var ent entitlements.Set = map[entitlements.Entitlement]entitlements.EntitlementsConfig{} err := b.EachValue(context.TODO(), keyEntitlements, func(v interface{}) error { set, ok := v.(entitlements.Set) if !ok { return errors.Errorf("invalid entitlements %T", v) } - for k := range set { - ent[k] = struct{}{} + for k, v := range set { + if prev, ok := ent[k]; ok && prev != nil { + prev.Merge(v) + } else { + ent[k] = v + } } return nil }) diff --git a/solver/llbsolver/vertex.go b/solver/llbsolver/vertex.go index 21ae0f9f22b8..c61f3b9ea719 100644 --- a/solver/llbsolver/vertex.go +++ b/solver/llbsolver/vertex.go @@ -7,6 +7,7 @@ import ( "github.com/containerd/platforms" "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/llbsolver/cdidevices" "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/apicaps" @@ -109,7 +110,7 @@ func NormalizeRuntimePlatforms() LoadOpt { } } -func ValidateEntitlements(ent entitlements.Set) LoadOpt { +func ValidateEntitlements(ent entitlements.Set, cdiManager *cdidevices.Manager) LoadOpt { return func(op *pb.Op, _ *pb.OpMetadata, opt *solver.VertexOptions) error { switch op := op.Op.(type) { case *pb.Op_Exec: @@ -120,6 +121,75 @@ func ValidateEntitlements(ent entitlements.Set) LoadOpt { if err := ent.Check(v); err != nil { return err } + if device := op.Exec.CdiDevices; len(device) > 0 { + var cfg *entitlements.DevicesConfig + if ent, ok := ent[entitlements.EntitlementDevice]; ok { + cfg, ok = ent.(*entitlements.DevicesConfig) + if !ok { + return errors.Errorf("invalid device entitlement config %T", ent) + } + } + if cfg != nil && cfg.All { + return nil + } + + var allowedDevices []*pb.CDIDevice + var nonAliasedDevices []*pb.CDIDevice + if cfg != nil { + for _, d := range op.Exec.CdiDevices { + if newName, ok := cfg.Devices[d.Name]; ok && newName != "" { + d.Name = newName + allowedDevices = append(allowedDevices, d) + } else { + nonAliasedDevices = append(nonAliasedDevices, d) + } + } + } else { + nonAliasedDevices = op.Exec.CdiDevices + } + + mountedDevices, err := cdiManager.FindDevices(nonAliasedDevices...) + if err != nil { + return err + } + if len(mountedDevices) == 0 { + op.Exec.CdiDevices = allowedDevices + return nil + } + + grantedDevices := make(map[string]struct{}) + if cfg != nil { + for d := range cfg.Devices { + resolved, err := cdiManager.FindDevices(&pb.CDIDevice{Name: d}) + if err != nil { + return err + } + for _, r := range resolved { + grantedDevices[r] = struct{}{} + } + } + } + + var forbidden []string + for _, d := range mountedDevices { + if _, ok := grantedDevices[d]; !ok { + if dev := cdiManager.GetDevice(d); !dev.AutoAllow { + forbidden = append(forbidden, d) + continue + } + } + allowedDevices = append(allowedDevices, &pb.CDIDevice{Name: d}) + } + + if len(forbidden) > 0 { + if len(forbidden) == 1 { + return errors.Errorf("device %s is requested by the build but not allowed", forbidden[0]) + } + return errors.Errorf("devices %s are requested by the build but not allowed", strings.Join(forbidden, ", ")) + } + + op.Exec.CdiDevices = allowedDevices + } } return nil } diff --git a/util/entitlements/entitlements.go b/util/entitlements/entitlements.go index 328580c326df..106f492ceee2 100644 --- a/util/entitlements/entitlements.go +++ b/util/entitlements/entitlements.go @@ -1,31 +1,119 @@ package entitlements import ( + "strings" + "github.com/pkg/errors" + "github.com/tonistiigi/go-csvvalue" ) type Entitlement string +func (e Entitlement) String() string { + return string(e) +} + const ( EntitlementSecurityInsecure Entitlement = "security.insecure" EntitlementNetworkHost Entitlement = "network.host" + EntitlementDevice Entitlement = "device" ) var all = map[Entitlement]struct{}{ EntitlementSecurityInsecure: {}, EntitlementNetworkHost: {}, + EntitlementDevice: {}, +} + +type EntitlementsConfig interface { + Merge(EntitlementsConfig) error } -func Parse(s string) (Entitlement, error) { +type DevicesConfig struct { + Devices map[string]string + All bool +} + +var _ EntitlementsConfig = &DevicesConfig{} + +func ParseDevicesConfig(s string) (*DevicesConfig, error) { + if s == "" { + return &DevicesConfig{All: true}, nil + } + + fields, err := csvvalue.Fields(s, nil) + if err != nil { + return nil, err + } + deviceName := fields[0] + var deviceAlias string + + for _, field := range fields[1:] { + k, v, ok := strings.Cut(field, "=") + if !ok { + return nil, errors.Errorf("invalid device config %q", field) + } + switch k { + case "alias": + deviceAlias = v + default: + return nil, errors.Errorf("unknown device config key %q", k) + } + } + + cfg := &DevicesConfig{Devices: map[string]string{}} + + if deviceAlias != "" { + cfg.Devices[deviceAlias] = deviceName + } else { + cfg.Devices[deviceName] = "" + } + return cfg, nil +} + +func (c *DevicesConfig) Merge(in EntitlementsConfig) error { + c2, ok := in.(*DevicesConfig) + if !ok { + return errors.Errorf("cannot merge %T into %T", in, c) + } + + if c2.All { + c.All = true + return nil + } + + for k, v := range c2.Devices { + if c.Devices == nil { + c.Devices = map[string]string{} + } + c.Devices[k] = v + } + return nil +} + +func Parse(s string) (Entitlement, EntitlementsConfig, error) { + var cfg EntitlementsConfig + key, rest, _ := strings.Cut(s, "=") + switch Entitlement(key) { + case EntitlementDevice: + s = key + var err error + cfg, err = ParseDevicesConfig(rest) + if err != nil { + return "", nil, err + } + default: + } + _, ok := all[Entitlement(s)] if !ok { - return "", errors.Errorf("unknown entitlement %s", s) + return "", nil, errors.Errorf("unknown entitlement %s", s) } - return Entitlement(s), nil + return Entitlement(s), cfg, nil } func WhiteList(allowed, supported []Entitlement) (Set, error) { - m := map[Entitlement]struct{}{} + m := map[Entitlement]EntitlementsConfig{} var supm Set if supported != nil { @@ -37,7 +125,7 @@ func WhiteList(allowed, supported []Entitlement) (Set, error) { } for _, e := range allowed { - e, err := Parse(string(e)) + e, cfg, err := Parse(string(e)) if err != nil { return nil, err } @@ -46,13 +134,19 @@ func WhiteList(allowed, supported []Entitlement) (Set, error) { return nil, errors.Errorf("granting entitlement %s is not allowed by build daemon configuration", e) } } - m[e] = struct{}{} + if prev, ok := m[e]; ok && prev != nil { + if err := prev.Merge(cfg); err != nil { + return nil, err + } + } else { + m[e] = cfg + } } return Set(m), nil } -type Set map[Entitlement]struct{} +type Set map[Entitlement]EntitlementsConfig func (s Set) Allowed(e Entitlement) bool { _, ok := s[e] @@ -77,4 +171,5 @@ func (s Set) Check(v Values) error { type Values struct { NetworkHost bool SecurityInsecure bool + Devices map[string]struct{} } diff --git a/worker/runc/runc.go b/worker/runc/runc.go index 0ea36169bdd2..1626530d5e7f 100644 --- a/worker/runc/runc.go +++ b/worker/runc/runc.go @@ -30,7 +30,6 @@ import ( ocispecs "github.com/opencontainers/image-spec/specs-go/v1" bolt "go.etcd.io/bbolt" "golang.org/x/sync/semaphore" - "tags.cncf.io/container-device-interface/pkg/cdi" ) // SnapshotterFactory instantiates a snapshotter @@ -40,7 +39,7 @@ type SnapshotterFactory struct { } // NewWorkerOpt creates a WorkerOpt. -func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, processMode oci.ProcessMode, labels map[string]string, idmap *idtools.IdentityMapping, nopt netproviders.Opt, dns *oci.DNSConfig, binary, apparmorProfile string, selinux bool, parallelismSem *semaphore.Weighted, traceSocket, defaultCgroupParent string, cdiManager *cdi.Cache) (base.WorkerOpt, error) { +func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, processMode oci.ProcessMode, labels map[string]string, idmap *idtools.IdentityMapping, nopt netproviders.Opt, dns *oci.DNSConfig, binary, apparmorProfile string, selinux bool, parallelismSem *semaphore.Weighted, traceSocket, defaultCgroupParent string, cdiManager *cdidevices.Manager) (base.WorkerOpt, error) { var opt base.WorkerOpt name := "runc-" + snFactory.Name root = filepath.Join(root, name) @@ -80,7 +79,7 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, proc TracingSocket: traceSocket, DefaultCgroupParent: defaultCgroupParent, ResourceMonitor: rm, - CDIManager: cdidevices.NewManager(cdiManager), + CDIManager: cdiManager, }, np) if err != nil { return opt, err @@ -169,7 +168,7 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, proc ParallelismSem: parallelismSem, MountPoolRoot: filepath.Join(root, "cachemounts"), ResourceMonitor: rm, - CDIManager: cdidevices.NewManager(cdiManager), + CDIManager: cdiManager, } return opt, nil } From e500309fc00b24825856d4035d7a432198a433fb Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Fri, 14 Feb 2025 09:52:15 +0100 Subject: [PATCH 03/23] cdi: keep auto refresh Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- cmd/buildkitd/main.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index 49e29943708e..364252a59b34 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -1052,7 +1052,6 @@ func newMeterProvider(ctx context.Context) (*sdkmetric.MeterProvider, error) { return sdkmetric.NewMeterProvider(opts...), nil } -// getCDIManager returns a new CDI registry with disabled auto-refresh. func getCDIManager(cfg config.CDIConfig) (*cdidevices.Manager, error) { if cfg.Disabled != nil && *cfg.Disabled { return nil, nil @@ -1061,10 +1060,7 @@ func getCDIManager(cfg config.CDIConfig) (*cdidevices.Manager, error) { return nil, errors.New("no CDI specification directories specified") } cdiCache, err := func() (*cdi.Cache, error) { - cdiCache, err := cdi.NewCache( - cdi.WithSpecDirs(cfg.SpecDirs...), - cdi.WithAutoRefresh(false), - ) + cdiCache, err := cdi.NewCache(cdi.WithSpecDirs(cfg.SpecDirs...)) if err != nil { return nil, err } From f901bcc6a8376c21e726f180039495883ffa7f21 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Fri, 14 Feb 2025 10:36:56 +0100 Subject: [PATCH 04/23] cdi: test find devices Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .../cdidevices/fixtures/vendor1-device.yaml | 9 +++ .../fixtures/vendor1-deviceclass.yaml | 28 ++++++++ .../fixtures/vendor1-devicemulti.yaml | 21 ++++++ solver/llbsolver/cdidevices/manager.go | 6 +- solver/llbsolver/cdidevices/manager_test.go | 70 +++++++++++++++++++ 5 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 solver/llbsolver/cdidevices/fixtures/vendor1-device.yaml create mode 100644 solver/llbsolver/cdidevices/fixtures/vendor1-deviceclass.yaml create mode 100644 solver/llbsolver/cdidevices/fixtures/vendor1-devicemulti.yaml create mode 100644 solver/llbsolver/cdidevices/manager_test.go diff --git a/solver/llbsolver/cdidevices/fixtures/vendor1-device.yaml b/solver/llbsolver/cdidevices/fixtures/vendor1-device.yaml new file mode 100644 index 000000000000..bf76973204c8 --- /dev/null +++ b/solver/llbsolver/cdidevices/fixtures/vendor1-device.yaml @@ -0,0 +1,9 @@ +cdiVersion: "0.6.0" +kind: "vendor1.com/device" +devices: + - name: foo + containerEdits: + env: + - FOO=injected +annotations: + org.mobyproject.buildkit.device.autoallow: true diff --git a/solver/llbsolver/cdidevices/fixtures/vendor1-deviceclass.yaml b/solver/llbsolver/cdidevices/fixtures/vendor1-deviceclass.yaml new file mode 100644 index 000000000000..2b2ee4b8cf45 --- /dev/null +++ b/solver/llbsolver/cdidevices/fixtures/vendor1-deviceclass.yaml @@ -0,0 +1,28 @@ +cdiVersion: "0.6.0" +kind: "vendor1.com/deviceclass" +annotations: + foo.bar.baz: FOO + org.mobyproject.buildkit.device.autoallow: true +devices: + - name: foo + annotations: + org.mobyproject.buildkit.device.class: class1 + containerEdits: + env: + - FOO=injected + - name: bar + annotations: + org.mobyproject.buildkit.device.class: class1 + containerEdits: + env: + - BAR=injected + - name: baz + annotations: + org.mobyproject.buildkit.device.class: class2 + containerEdits: + env: + - BAZ=injected + - name: qux + containerEdits: + env: + - QUX=injected diff --git a/solver/llbsolver/cdidevices/fixtures/vendor1-devicemulti.yaml b/solver/llbsolver/cdidevices/fixtures/vendor1-devicemulti.yaml new file mode 100644 index 000000000000..f6940569ddcb --- /dev/null +++ b/solver/llbsolver/cdidevices/fixtures/vendor1-devicemulti.yaml @@ -0,0 +1,21 @@ +cdiVersion: "0.6.0" +kind: "vendor1.com/devicemulti" +devices: + - name: foo + containerEdits: + env: + - FOO=injected + - name: bar + containerEdits: + env: + - BAR=injected + - name: baz + containerEdits: + env: + - BAZ=injected + - name: qux + containerEdits: + env: + - QUX=injected +annotations: + org.mobyproject.buildkit.device.autoallow: true diff --git a/solver/llbsolver/cdidevices/manager.go b/solver/llbsolver/cdidevices/manager.go index 2f207d0e5435..673c823f25c8 100644 --- a/solver/llbsolver/cdidevices/manager.go +++ b/solver/llbsolver/cdidevices/manager.go @@ -15,8 +15,8 @@ import ( ) const ( - deviceAnnotationClass = "org.mobyproject.buildkit.device.class" - deviceAutoAllow = "org.mobyproject.buildkit.device.autoallow" + deviceAnnotationClass = "org.mobyproject.buildkit.device.class" + deviceAnnotationAutoAllow = "org.mobyproject.buildkit.device.autoallow" ) var installers = map[string]Setup{} @@ -63,7 +63,7 @@ func (m *Manager) isAutoAllowed(kind, name string, annotations map[string]string if _, ok := m.autoAllowed[kind]; ok { return true } - if v, ok := annotations[deviceAutoAllow]; ok { + if v, ok := annotations[deviceAnnotationAutoAllow]; ok { if b, err := strconv.ParseBool(v); err == nil && b { return true } diff --git a/solver/llbsolver/cdidevices/manager_test.go b/solver/llbsolver/cdidevices/manager_test.go new file mode 100644 index 000000000000..057a17811bcb --- /dev/null +++ b/solver/llbsolver/cdidevices/manager_test.go @@ -0,0 +1,70 @@ +package cdidevices + +import ( + "testing" + + "github.com/moby/buildkit/solver/pb" + "github.com/stretchr/testify/require" + "tags.cncf.io/container-device-interface/pkg/cdi" +) + +func TestFindDevices(t *testing.T) { + tests := []struct { + name string + devices []*pb.CDIDevice + expected []string + err bool + }{ + { + name: "Find specific device", + devices: []*pb.CDIDevice{ + {Name: "vendor1.com/device=foo"}, + }, + expected: []string{"vendor1.com/device=foo"}, + }, + { + name: "Find first devices", + devices: []*pb.CDIDevice{ + {Name: "vendor1.com/devicemulti"}, + }, + expected: []string{"vendor1.com/devicemulti=bar"}, + }, + { + name: "Find all devices of a kind", + devices: []*pb.CDIDevice{ + {Name: "vendor1.com/devicemulti=*"}, + }, + expected: []string{"vendor1.com/devicemulti=foo", "vendor1.com/devicemulti=bar", "vendor1.com/devicemulti=baz", "vendor1.com/devicemulti=qux"}, + }, + { + name: "Find devices by class", + devices: []*pb.CDIDevice{ + {Name: "vendor1.com/deviceclass=class1"}, + }, + expected: []string{"vendor1.com/deviceclass=foo", "vendor1.com/deviceclass=bar"}, + }, + { + name: "Device not found", + devices: []*pb.CDIDevice{ + {Name: "vendor1.com/device=unknown"}, + }, + expected: nil, + err: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cache, err := cdi.NewCache(cdi.WithSpecDirs("./fixtures")) + require.NoError(t, err) + manager := NewManager(cache, nil) + result, err := manager.FindDevices(tt.devices...) + if tt.err { + require.Error(t, err) + } else { + require.NoError(t, err) + require.ElementsMatch(t, tt.expected, result) + } + }) + } +} From 281e8c9d0ef85ff42f3c921673c36aeb0b28508c Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Fri, 14 Feb 2025 11:02:00 +0100 Subject: [PATCH 05/23] client: test cdi entitlement Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- client/client_test.go | 100 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/client/client_test.go b/client/client_test.go index 5b79d680e230..f266b5bac2ac 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -278,6 +278,8 @@ func testIntegration(t *testing.T, funcs ...func(t *testing.T, sb integration.Sa integration.Run(t, integration.TestFuncs( testCDI, + testCDINotAllowed, + testCDIEntitlement, testCDIFirst, testCDIWildcard, testCDIClass, @@ -11111,6 +11113,104 @@ annotations: require.Contains(t, strings.TrimSpace(string(dt2)), `BAR=injected`) } +func testCDINotAllowed(t *testing.T, sb integration.Sandbox) { + if sb.Rootless() { + t.SkipNow() + } + + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureCDI) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(` +cdiVersion: "0.6.0" +kind: "vendor1.com/device" +devices: +- name: foo + containerEdits: + env: + - FOO=injected +`), 0600)) + + busybox := llb.Image("busybox:latest") + st := llb.Scratch() + + run := func(cmd string, ro ...llb.RunOption) { + st = busybox.Run(append(ro, llb.Shlex(cmd), llb.Dir("/wd"))...).AddMount("/wd", st) + } + + run(`sh -c 'env|sort | tee foo.env'`, llb.AddCDIDevice(llb.CDIDeviceName("vendor1.com/device=foo"))) + + def, err := st.Marshal(sb.Context()) + require.NoError(t, err) + + destDir := t.TempDir() + + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.Error(t, err) + require.ErrorContains(t, err, "requested by the build but not allowed") +} + +func testCDIEntitlement(t *testing.T, sb integration.Sandbox) { + if sb.Rootless() { + t.SkipNow() + } + + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureCDI) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(` +cdiVersion: "0.6.0" +kind: "vendor1.com/device" +devices: +- name: foo + containerEdits: + env: + - FOO=injected +`), 0600)) + + busybox := llb.Image("busybox:latest") + st := llb.Scratch() + + run := func(cmd string, ro ...llb.RunOption) { + st = busybox.Run(append(ro, llb.Shlex(cmd), llb.Dir("/wd"))...).AddMount("/wd", st) + } + + run(`sh -c 'env|sort | tee foo.env'`, llb.AddCDIDevice(llb.CDIDeviceName("vendor1.com/device=foo"))) + + def, err := st.Marshal(sb.Context()) + require.NoError(t, err) + + destDir := t.TempDir() + + _, err = c.Solve(sb.Context(), def, SolveOpt{ + AllowedEntitlements: []string{"device=vendor1.com/device"}, + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(filepath.Join(destDir, "foo.env")) + require.NoError(t, err) + require.Contains(t, strings.TrimSpace(string(dt)), `FOO=injected`) +} + func testCDIFirst(t *testing.T, sb integration.Sandbox) { if sb.Rootless() { t.SkipNow() From 61f13c08632830f8026d94676fc03607245874a1 Mon Sep 17 00:00:00 2001 From: Austin Vazquez Date: Fri, 14 Feb 2025 04:01:16 +0000 Subject: [PATCH 06/23] dockerfile: update runc to 1.2.5 Signed-off-by: Austin Vazquez --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5e57fd48e79a..6775afc65254 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile-upstream:master -ARG RUNC_VERSION=v1.2.4 +ARG RUNC_VERSION=v1.2.5 ARG CONTAINERD_VERSION=v2.0.2 # CONTAINERD_ALT_VERSION_... defines fallback containerd version for integration tests ARG CONTAINERD_ALT_VERSION_17=v1.7.25 From 5ae6c31b11eaa914efc79bd3cdbe6e03852c3b8a Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 18 Feb 2025 15:00:00 +0100 Subject: [PATCH 07/23] buildctl: set fallback url for gha cache Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- README.md | 3 ++- cmd/buildctl/build/util.go | 25 +++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 4a9294b2dd93..ee5293b1e5ba 100644 --- a/README.md +++ b/README.md @@ -516,7 +516,8 @@ GitHub Actions cache saves both cache metadata and layers to GitHub's Cache serv Similarly to using [actions/cache](https://github.com/actions/cache), caches are [scoped by branch](https://docs.github.com/en/actions/advanced-guides/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache), with the default and target branches being available to every branch. Following attributes are required to authenticate against the [GitHub Actions Cache service API](https://github.com/tonistiigi/go-actions-cache/blob/master/api.md#authentication): -* `url`: Cache server URL (default `$ACTIONS_CACHE_URL`) +* `url`: Cache server URL (default `$ACTIONS_CACHE_URL` or fallback to `$ACTIONS_RESULTS_URL`) +* `url_v2`: Cache v2 server URL if `$ACTIONS_CACHE_SERVICE_V2` set on the runner (default `$ACTIONS_RESULTS_URL`) * `token`: Access token (default `$ACTIONS_RUNTIME_TOKEN`) :information_source: This type of cache can be used with [Docker Build Push Action](https://github.com/docker/build-push-action) diff --git a/cmd/buildctl/build/util.go b/cmd/buildctl/build/util.go index 62a4398431c4..7897ad04f617 100644 --- a/cmd/buildctl/build/util.go +++ b/cmd/buildctl/build/util.go @@ -4,10 +4,9 @@ import ( "os" "strconv" - "github.com/pkg/errors" - "github.com/moby/buildkit/client" "github.com/moby/buildkit/util/bklog" + "github.com/pkg/errors" ) // loadGithubEnv verify that url and token attributes exists in the @@ -18,6 +17,7 @@ import ( func loadGithubEnv(cache client.CacheOptionsEntry) (client.CacheOptionsEntry, error) { version, ok := cache.Attrs["version"] if !ok { + // https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L19 if v, ok := os.LookupEnv("ACTIONS_CACHE_SERVICE_V2"); ok { if b, err := strconv.ParseBool(v); err == nil && b { version = "2" @@ -26,18 +26,23 @@ func loadGithubEnv(cache client.CacheOptionsEntry) (client.CacheOptionsEntry, er } if _, ok := cache.Attrs["url_v2"]; !ok && version == "2" { - url, ok := os.LookupEnv("ACTIONS_RESULTS_URL") - if !ok { - return cache, errors.New("cache with type gha requires url parameter or $ACTIONS_RESULTS_URL") + // https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L34-L35 + if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok { + cache.Attrs["url_v2"] = v } - cache.Attrs["url_v2"] = url } if _, ok := cache.Attrs["url"]; !ok { - url, ok := os.LookupEnv("ACTIONS_CACHE_URL") - if !ok { - return cache, errors.New("cache with type gha requires url parameter or $ACTIONS_CACHE_URL") + // https://github.com/actions/toolkit/blob/2b08dc18f261b9fdd978b70279b85cbef81af8bc/packages/cache/src/internal/config.ts#L28-L33 + if v, ok := os.LookupEnv("ACTIONS_CACHE_URL"); ok { + cache.Attrs["url"] = v + } else if v, ok := os.LookupEnv("ACTIONS_RESULTS_URL"); ok { + cache.Attrs["url"] = v + } + } + if _, ok := cache.Attrs["url"]; !ok { + if _, ok := cache.Attrs["url_v2"]; !ok { + return cache, errors.New("cache with type gha requires url parameter to be set") } - cache.Attrs["url"] = url } if _, ok := cache.Attrs["token"]; !ok { From b0f75aa807a79cee73d0500864144ab5fdf66cea Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Tue, 18 Feb 2025 09:54:54 +0100 Subject: [PATCH 08/23] test: handle gha cache v2 Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- cache/remotecache/gha/gha_test.go | 53 +++++++++++++++++--------- cmd/buildctl/build/importcache_test.go | 25 ++++++++++-- hack/test | 2 + 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/cache/remotecache/gha/gha_test.go b/cache/remotecache/gha/gha_test.go index fd92a14eebe8..6da0e85da44c 100644 --- a/cache/remotecache/gha/gha_test.go +++ b/cache/remotecache/gha/gha_test.go @@ -1,8 +1,10 @@ package gha import ( + "maps" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -57,10 +59,25 @@ func testBasicGhaCacheImportExportExtraTimeout(t *testing.T, sb integration.Sand destDir := t.TempDir() - runtimeToken := os.Getenv("ACTIONS_RUNTIME_TOKEN") - cacheURL := os.Getenv("ACTIONS_CACHE_URL") - if runtimeToken == "" || cacheURL == "" { - t.Skip("ACTIONS_RUNTIME_TOKEN and ACTIONS_CACHE_URL must be set") + var cacheVersion string + if v, ok := os.LookupEnv("ACTIONS_CACHE_SERVICE_V2"); ok { + if b, err := strconv.ParseBool(v); err == nil && b { + cacheVersion = "2" + } + } + + cacheAttrs := map[string]string{} + if cacheVersion == "2" { + cacheAttrs["url_v2"] = os.Getenv("ACTIONS_RESULTS_URL") + } + cacheAttrs["url"] = os.Getenv("ACTIONS_CACHE_URL") + if cacheAttrs["url"] == "" { + cacheAttrs["url"] = os.Getenv("ACTIONS_RESULTS_URL") + } + cacheAttrs["token"] = os.Getenv("ACTIONS_RUNTIME_TOKEN") + + if cacheAttrs["token"] == "" || (cacheAttrs["url"] == "" && cacheAttrs["url_v2"] == "") { + t.Skip("actions runtime token and cache url must be set") } scope := "buildkit-" + t.Name() @@ -74,6 +91,12 @@ func testBasicGhaCacheImportExportExtraTimeout(t *testing.T, sb integration.Sand } } + cacheExportAttrs := map[string]string{ + "scope": scope, + "mode": "max", + } + maps.Copy(cacheExportAttrs, cacheAttrs) + _, err = c.Solve(sb.Context(), def, client.SolveOpt{ Exports: []client.ExportEntry{ { @@ -82,13 +105,8 @@ func testBasicGhaCacheImportExportExtraTimeout(t *testing.T, sb integration.Sand }, }, CacheExports: []client.CacheOptionsEntry{{ - Type: "gha", - Attrs: map[string]string{ - "url": cacheURL, - "token": runtimeToken, - "scope": scope, - "mode": "max", - }, + Type: "gha", + Attrs: cacheExportAttrs, }}, }, nil) require.NoError(t, err) @@ -104,6 +122,11 @@ func testBasicGhaCacheImportExportExtraTimeout(t *testing.T, sb integration.Sand destDir = t.TempDir() + cacheImportAttrs := map[string]string{ + "scope": scope, + } + maps.Copy(cacheImportAttrs, cacheAttrs) + _, err = c.Solve(sb.Context(), def, client.SolveOpt{ Exports: []client.ExportEntry{ { @@ -112,12 +135,8 @@ func testBasicGhaCacheImportExportExtraTimeout(t *testing.T, sb integration.Sand }, }, CacheImports: []client.CacheOptionsEntry{{ - Type: "gha", - Attrs: map[string]string{ - "url": cacheURL, - "token": runtimeToken, - "scope": scope, - }, + Type: "gha", + Attrs: cacheImportAttrs, }}, }, nil) require.NoError(t, err) diff --git a/cmd/buildctl/build/importcache_test.go b/cmd/buildctl/build/importcache_test.go index ad175812d65e..bc8c1056089f 100644 --- a/cmd/buildctl/build/importcache_test.go +++ b/cmd/buildctl/build/importcache_test.go @@ -54,8 +54,22 @@ func TestParseImportCache(t *testing.T) { { Type: "gha", Attrs: map[string]string{ - "url": "https://foo.bar", - "token": "foo", + "url": "https://foo.bar", + "url_v2": "https://github.com/testv2", // Set from env below + "token": "foo", + }, + }, + }, + }, + { + importCaches: []string{"type=gha,url_v2=https://foo.bar,token=foo"}, + expected: []client.CacheOptionsEntry{ + { + Type: "gha", + Attrs: map[string]string{ + "url": "https://github.com/test", // Set from env below + "url_v2": "https://foo.bar", + "token": "foo", }, }, }, @@ -66,8 +80,9 @@ func TestParseImportCache(t *testing.T) { { Type: "gha", Attrs: map[string]string{ - "url": "https://github.com/test", // Set from env below - "token": "bar", // Set from env below + "url": "https://github.com/test", // Set from env below + "url_v2": "https://github.com/testv2", // Set from env below + "token": "bar", // Set from env below }, }, }, @@ -75,7 +90,9 @@ func TestParseImportCache(t *testing.T) { } // Set values for GitHub parse cache + t.Setenv("ACTIONS_CACHE_SERVICE_V2", "True") t.Setenv("ACTIONS_CACHE_URL", "https://github.com/test") + t.Setenv("ACTIONS_RESULTS_URL", "https://github.com/testv2") t.Setenv("ACTIONS_RUNTIME_TOKEN", "bar") for _, tc := range testCases { diff --git a/hack/test b/hack/test index 85f07557b10d..0342ee94014a 100755 --- a/hack/test +++ b/hack/test @@ -125,7 +125,9 @@ baseCreateFlags="--rm --privileged $dockerConfigMount \ -e CGO_ENABLED \ -e GITHUB_REF \ -e ACTIONS_RUNTIME_TOKEN \ +-e ACTIONS_CACHE_SERVICE_V2 \ -e ACTIONS_CACHE_URL \ +-e ACTIONS_RESULTS_URL \ -e TEST_DOCKERD \ -e BUILDKIT_TEST_ENABLE_FEATURES \ -e BUILDKIT_TEST_DISABLE_FEATURES \ From c016cdacba9ae85c7f4303e3fb5b5c29ab204f62 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 19 Feb 2025 12:55:42 +0100 Subject: [PATCH 09/23] cache(gha): set user-agent for github cache service requests Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- cache/remotecache/gha/gha.go | 6 ++++-- go.mod | 2 +- go.sum | 4 ++-- vendor/github.com/tonistiigi/go-actions-cache/cache.go | 7 +++++++ vendor/github.com/tonistiigi/go-actions-cache/cache_v2.go | 1 + vendor/modules.txt | 2 +- 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cache/remotecache/gha/gha.go b/cache/remotecache/gha/gha.go index a7af088e752a..8acf15acfa83 100644 --- a/cache/remotecache/gha/gha.go +++ b/cache/remotecache/gha/gha.go @@ -22,6 +22,7 @@ import ( "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/tracing" + bkversion "github.com/moby/buildkit/version" "github.com/moby/buildkit/worker" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" @@ -140,8 +141,9 @@ type exporter struct { func NewExporter(c *Config) (remotecache.Exporter, error) { cc := v1.NewCacheChains() cache, err := actionscache.New(c.Token, c.URL, c.Version > 1, actionscache.Opt{ - Client: tracing.DefaultClient, - Timeout: c.Timeout, + Client: tracing.DefaultClient, + Timeout: c.Timeout, + UserAgent: bkversion.UserAgent(), }) if err != nil { return nil, err diff --git a/go.mod b/go.mod index af73f4a40c75..fe5a805c6cca 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/stretchr/testify v1.10.0 github.com/tonistiigi/dchapes-mode v0.0.0-20241001053921-ca0759fec205 github.com/tonistiigi/fsutil v0.0.0-20250113203817-b14e27f4135a - github.com/tonistiigi/go-actions-cache v0.0.0-20250211194249-bd99cf5bbc65 + github.com/tonistiigi/go-actions-cache v0.0.0-20250219102945-1a5174abd055 github.com/tonistiigi/go-archvariant v1.0.0 github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea diff --git a/go.sum b/go.sum index 5b587f07a1ea..ce10e2f751c0 100644 --- a/go.sum +++ b/go.sum @@ -401,8 +401,8 @@ github.com/tonistiigi/dchapes-mode v0.0.0-20241001053921-ca0759fec205 h1:eUk79E1 github.com/tonistiigi/dchapes-mode v0.0.0-20241001053921-ca0759fec205/go.mod h1:3Iuxbr0P7D3zUzBMAZB+ois3h/et0shEz0qApgHYGpY= github.com/tonistiigi/fsutil v0.0.0-20250113203817-b14e27f4135a h1:EfGw4G0x/8qXWgtcZ6KVaPS+wpWOQMaypczzP8ojkMY= github.com/tonistiigi/fsutil v0.0.0-20250113203817-b14e27f4135a/go.mod h1:Dl/9oEjK7IqnjAm21Okx/XIxUCFJzvh+XdVHUlBwXTw= -github.com/tonistiigi/go-actions-cache v0.0.0-20250211194249-bd99cf5bbc65 h1:57xLt2zJ6in5Au6plQUKm1gfasse4j3h9lrvoor2xPs= -github.com/tonistiigi/go-actions-cache v0.0.0-20250211194249-bd99cf5bbc65/go.mod h1:h0oRlVs3NoFIHysRQ4rU1+RG4QmU0M2JVSwTYrB4igk= +github.com/tonistiigi/go-actions-cache v0.0.0-20250219102945-1a5174abd055 h1:ocuqZe/ImPTYgDocnHTDOTBachfQ9m0JxvP4uGeDtBE= +github.com/tonistiigi/go-actions-cache v0.0.0-20250219102945-1a5174abd055/go.mod h1:h0oRlVs3NoFIHysRQ4rU1+RG4QmU0M2JVSwTYrB4igk= github.com/tonistiigi/go-archvariant v1.0.0 h1:5LC1eDWiBNflnTF1prCiX09yfNHIxDC/aukdhCdTyb0= github.com/tonistiigi/go-archvariant v1.0.0/go.mod h1:TxFmO5VS6vMq2kvs3ht04iPXtu2rUT/erOnGFYfk5Ho= github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 h1:7I5c2Ig/5FgqkYOh/N87NzoyI9U15qUPXhDD8uCupv8= diff --git a/vendor/github.com/tonistiigi/go-actions-cache/cache.go b/vendor/github.com/tonistiigi/go-actions-cache/cache.go index a3173e3d4192..a47fc9cc4285 100644 --- a/vendor/github.com/tonistiigi/go-actions-cache/cache.go +++ b/vendor/github.com/tonistiigi/go-actions-cache/cache.go @@ -27,6 +27,8 @@ var UploadConcurrency = 4 var UploadChunkSize = 32 * 1024 * 1024 var noValidateToken bool +const defaultUserAgent = "go-actions-cache/1.0" + var Log = func(string, ...interface{}) {} type Blob interface { @@ -110,6 +112,7 @@ type Opt struct { Client *http.Client Timeout time.Duration BackoffPool *BackoffPool + UserAgent string } func New(token, url string, v2 bool, opt Opt) (*Cache, error) { @@ -187,6 +190,9 @@ func optsWithDefaults(opt Opt) Opt { if opt.BackoffPool == nil { opt.BackoffPool = defaultBackoffPool } + if opt.UserAgent == "" { + opt.UserAgent = defaultUserAgent + } return opt } @@ -495,6 +501,7 @@ func (c *Cache) newRequest(method, url string, body func() io.Reader) *request { headers: map[string]string{ "Authorization": "Bearer " + c.Token.Raw, "Accept": "application/json;api-version=6.0-preview.1", + "User-Agent": c.opt.UserAgent, }, } } diff --git a/vendor/github.com/tonistiigi/go-actions-cache/cache_v2.go b/vendor/github.com/tonistiigi/go-actions-cache/cache_v2.go index 22ea996e4a26..adbcda30eb68 100644 --- a/vendor/github.com/tonistiigi/go-actions-cache/cache_v2.go +++ b/vendor/github.com/tonistiigi/go-actions-cache/cache_v2.go @@ -185,6 +185,7 @@ func (c *Cache) newRequestV2(url string, body func() io.Reader) *request { headers: map[string]string{ "Authorization": "Bearer " + c.Token.Raw, "Content-Type": "application/json", + "User-Agent": c.opt.UserAgent, }, } } diff --git a/vendor/modules.txt b/vendor/modules.txt index d34005216727..5f5ee4e5da65 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -809,7 +809,7 @@ github.com/tonistiigi/dchapes-mode github.com/tonistiigi/fsutil github.com/tonistiigi/fsutil/copy github.com/tonistiigi/fsutil/types -# github.com/tonistiigi/go-actions-cache v0.0.0-20250211194249-bd99cf5bbc65 +# github.com/tonistiigi/go-actions-cache v0.0.0-20250219102945-1a5174abd055 ## explicit; go 1.22 github.com/tonistiigi/go-actions-cache # github.com/tonistiigi/go-archvariant v1.0.0 From c7153d1eb5912e5ae42571559bd334374f325802 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:34:25 +0100 Subject: [PATCH 10/23] cache(gha): fix missing user-agent for importer Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- cache/remotecache/gha/gha.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cache/remotecache/gha/gha.go b/cache/remotecache/gha/gha.go index 8acf15acfa83..c7ef08be73dc 100644 --- a/cache/remotecache/gha/gha.go +++ b/cache/remotecache/gha/gha.go @@ -317,8 +317,9 @@ type importer struct { func NewImporter(c *Config) (remotecache.Importer, error) { cache, err := actionscache.New(c.Token, c.URL, c.Version > 1, actionscache.Opt{ - Client: tracing.DefaultClient, - Timeout: c.Timeout, + Client: tracing.DefaultClient, + Timeout: c.Timeout, + UserAgent: bkversion.UserAgent(), }) if err != nil { return nil, err From 547d0836b26071997a30b8a53bfabfa7b05524b2 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sat, 22 Feb 2025 08:08:57 -0800 Subject: [PATCH 11/23] vendor: update cdi to v0.8.1 for panic fix Signed-off-by: Tonis Tiigi --- cmd/buildkitd/main.go | 7 +++++++ go.mod | 2 +- go.sum | 4 ++-- vendor/modules.txt | 2 +- .../container-device-interface/pkg/cdi/cache.go | 8 ++++++++ 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index 364252a59b34..8b6e0b3653c8 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -1067,6 +1067,13 @@ func getCDIManager(cfg config.CDIConfig) (*cdidevices.Manager, error) { if err := cdiCache.Refresh(); err != nil { return nil, err } + if errs := cdiCache.GetErrors(); len(errs) > 0 { + for dir, errs := range errs { + for _, err := range errs { + bklog.L.Warnf("CDI setup error %v: %+v", dir, err) + } + } + } return cdiCache, nil }() if err != nil { diff --git a/go.mod b/go.mod index fe5a805c6cca..7236c50f0d7d 100644 --- a/go.mod +++ b/go.mod @@ -108,7 +108,7 @@ require ( google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 google.golang.org/protobuf v1.35.2 kernel.org/pub/linux/libs/security/libcap/cap v1.2.73 - tags.cncf.io/container-device-interface v0.8.0 + tags.cncf.io/container-device-interface v0.8.1 ) require ( diff --git a/go.sum b/go.sum index ce10e2f751c0..2323ccd7f5e5 100644 --- a/go.sum +++ b/go.sum @@ -594,7 +594,7 @@ kernel.org/pub/linux/libs/security/libcap/psx v1.2.73/go.mod h1:+l6Ee2F59XiJ2I6W sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= -tags.cncf.io/container-device-interface v0.8.0 h1:8bCFo/g9WODjWx3m6EYl3GfUG31eKJbaggyBDxEldRc= -tags.cncf.io/container-device-interface v0.8.0/go.mod h1:Apb7N4VdILW0EVdEMRYXIDVRZfNJZ+kmEUss2kRRQ6Y= +tags.cncf.io/container-device-interface v0.8.1 h1:c0jN4Mt6781jD67NdPajmZlD1qrqQyov/Xfoab37lj0= +tags.cncf.io/container-device-interface v0.8.1/go.mod h1:Apb7N4VdILW0EVdEMRYXIDVRZfNJZ+kmEUss2kRRQ6Y= tags.cncf.io/container-device-interface/specs-go v0.8.0 h1:QYGFzGxvYK/ZLMrjhvY0RjpUavIn4KcmRmVP/JjdBTA= tags.cncf.io/container-device-interface/specs-go v0.8.0/go.mod h1:BhJIkjjPh4qpys+qm4DAYtUyryaTDg9zris+AczXyws= diff --git a/vendor/modules.txt b/vendor/modules.txt index 5f5ee4e5da65..12e99ac6987c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1148,7 +1148,7 @@ kernel.org/pub/linux/libs/security/libcap/psx ## explicit; go 1.12 sigs.k8s.io/yaml sigs.k8s.io/yaml/goyaml.v2 -# tags.cncf.io/container-device-interface v0.8.0 +# tags.cncf.io/container-device-interface v0.8.1 ## explicit; go 1.20 tags.cncf.io/container-device-interface/internal/validation tags.cncf.io/container-device-interface/internal/validation/k8s diff --git a/vendor/tags.cncf.io/container-device-interface/pkg/cdi/cache.go b/vendor/tags.cncf.io/container-device-interface/pkg/cdi/cache.go index c2f7fe346376..9afa4b18202e 100644 --- a/vendor/tags.cncf.io/container-device-interface/pkg/cdi/cache.go +++ b/vendor/tags.cncf.io/container-device-interface/pkg/cdi/cache.go @@ -564,6 +564,14 @@ func (w *watch) update(dirErrors map[string]error, removed ...string) bool { update bool ) + // If we failed to create an fsnotify.Watcher we have a nil watcher here + // (but with autoRefresh left on). One known case when this can happen is + // if we have too many open files. In that case we always return true and + // force a refresh. + if w.watcher == nil { + return true + } + for dir, ok = range w.tracked { if ok { continue From b15731bd35db82e3cee16697bf5b58e04f9d543e Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Fri, 28 Feb 2025 15:18:30 -0800 Subject: [PATCH 12/23] vendor: update action-cache with longer azure retries Signed-off-by: Tonis Tiigi --- go.mod | 2 +- go.sum | 4 ++-- .../tonistiigi/go-actions-cache/cache_v2.go | 17 +++++++++++++++-- vendor/modules.txt | 2 +- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 7236c50f0d7d..166697f4ac4e 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/stretchr/testify v1.10.0 github.com/tonistiigi/dchapes-mode v0.0.0-20241001053921-ca0759fec205 github.com/tonistiigi/fsutil v0.0.0-20250113203817-b14e27f4135a - github.com/tonistiigi/go-actions-cache v0.0.0-20250219102945-1a5174abd055 + github.com/tonistiigi/go-actions-cache v0.0.0-20250228231703-3e9a6642607f github.com/tonistiigi/go-archvariant v1.0.0 github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea diff --git a/go.sum b/go.sum index 2323ccd7f5e5..fa32deebbfc0 100644 --- a/go.sum +++ b/go.sum @@ -401,8 +401,8 @@ github.com/tonistiigi/dchapes-mode v0.0.0-20241001053921-ca0759fec205 h1:eUk79E1 github.com/tonistiigi/dchapes-mode v0.0.0-20241001053921-ca0759fec205/go.mod h1:3Iuxbr0P7D3zUzBMAZB+ois3h/et0shEz0qApgHYGpY= github.com/tonistiigi/fsutil v0.0.0-20250113203817-b14e27f4135a h1:EfGw4G0x/8qXWgtcZ6KVaPS+wpWOQMaypczzP8ojkMY= github.com/tonistiigi/fsutil v0.0.0-20250113203817-b14e27f4135a/go.mod h1:Dl/9oEjK7IqnjAm21Okx/XIxUCFJzvh+XdVHUlBwXTw= -github.com/tonistiigi/go-actions-cache v0.0.0-20250219102945-1a5174abd055 h1:ocuqZe/ImPTYgDocnHTDOTBachfQ9m0JxvP4uGeDtBE= -github.com/tonistiigi/go-actions-cache v0.0.0-20250219102945-1a5174abd055/go.mod h1:h0oRlVs3NoFIHysRQ4rU1+RG4QmU0M2JVSwTYrB4igk= +github.com/tonistiigi/go-actions-cache v0.0.0-20250228231703-3e9a6642607f h1:q/SWz3Bz0KtAsqaBo73CHVXjaz5O8PDnmD2JHVhgYnE= +github.com/tonistiigi/go-actions-cache v0.0.0-20250228231703-3e9a6642607f/go.mod h1:h0oRlVs3NoFIHysRQ4rU1+RG4QmU0M2JVSwTYrB4igk= github.com/tonistiigi/go-archvariant v1.0.0 h1:5LC1eDWiBNflnTF1prCiX09yfNHIxDC/aukdhCdTyb0= github.com/tonistiigi/go-archvariant v1.0.0/go.mod h1:TxFmO5VS6vMq2kvs3ht04iPXtu2rUT/erOnGFYfk5Ho= github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 h1:7I5c2Ig/5FgqkYOh/N87NzoyI9U15qUPXhDD8uCupv8= diff --git a/vendor/github.com/tonistiigi/go-actions-cache/cache_v2.go b/vendor/github.com/tonistiigi/go-actions-cache/cache_v2.go index adbcda30eb68..6bf7ec3c084e 100644 --- a/vendor/github.com/tonistiigi/go-actions-cache/cache_v2.go +++ b/vendor/github.com/tonistiigi/go-actions-cache/cache_v2.go @@ -5,7 +5,10 @@ import ( "context" "encoding/json" "io" + "time" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" "github.com/pkg/errors" @@ -45,8 +48,18 @@ func (c *Cache) reserveV2(ctx context.Context, key string) (string, error) { return cr.SignedUploadURL, nil } +var azureOptions = &blockblob.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Retry: policy.RetryOptions{ + MaxRetries: 10, + MaxRetryDelay: 2 * time.Minute, + RetryDelay: 10 * time.Second, + }, + }, +} + func (c *Cache) uploadV2(ctx context.Context, url string, b Blob) error { - client, err := blockblob.NewClientWithNoCredential(url, nil) + client, err := blockblob.NewClientWithNoCredential(url, azureOptions) if err != nil { return errors.WithStack(err) } @@ -62,7 +75,7 @@ func (c *Cache) uploadV2(ctx context.Context, url string, b Blob) error { func (ce *Entry) downloadV2(ctx context.Context) ReaderAtCloser { return toReaderAtCloser(func(offset int64) (io.ReadCloser, error) { - client, err := blockblob.NewClientWithNoCredential(ce.URL, nil) + client, err := blockblob.NewClientWithNoCredential(ce.URL, azureOptions) if err != nil { return nil, errors.WithStack(err) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 12e99ac6987c..e95c71415741 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -809,7 +809,7 @@ github.com/tonistiigi/dchapes-mode github.com/tonistiigi/fsutil github.com/tonistiigi/fsutil/copy github.com/tonistiigi/fsutil/types -# github.com/tonistiigi/go-actions-cache v0.0.0-20250219102945-1a5174abd055 +# github.com/tonistiigi/go-actions-cache v0.0.0-20250228231703-3e9a6642607f ## explicit; go 1.22 github.com/tonistiigi/go-actions-cache # github.com/tonistiigi/go-archvariant v1.0.0 From b1b594eec53c1b8244432db6ccc36a2d1a50802a Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 3 Mar 2025 15:58:11 -0800 Subject: [PATCH 13/23] update binfmt to v9.2.2 Should help with segmentation fault on libc-bin Signed-off-by: Tonis Tiigi --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6775afc65254..50a21cdb2d57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -162,8 +162,8 @@ FROM scratch AS cni-plugins-export-squashed COPY --from=cni-plugins-export / / FROM --platform=$BUILDPLATFORM alpine:${ALPINE_VERSION} AS binfmt-filter -# built from https://github.com/tonistiigi/binfmt/releases/tag/buildkit%2Fv9.2.0-50 -COPY --link --from=tonistiigi/binfmt:buildkit-v9.2.0-50@sha256:ff21b00e7238dce3bbd74fbe25591f7213837a77861b47b2df5e019540ec33fa / /out/ +# built from https://github.com/tonistiigi/binfmt/releases/tag/buildkit%2Fv9.2.2-54 +COPY --link --from=tonistiigi/binfmt:buildkit-v9.2.2-54@sha256:e60fbf01e26c75efa816224f4de31c2ef63c5486b20c3e8fa1e5da2aff368ba9 / /out/ WORKDIR /out/ RUN rm buildkit-qemu-loongarch64 buildkit-qemu-mips64 buildkit-qemu-mips64el From bfe3c1730a4256a509cbe07390da48de9504975d Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Fri, 21 Feb 2025 14:50:24 +0900 Subject: [PATCH 14/23] rootless: update docs and examples Fix issue 5763 - Discourage `--oci-worker-no-process-sandbox`, due to the leakage of the processes (by design). Instead, encourage setting `systempaths=unconfined` in `docker run`. This corresponds to `securityContext.procMount: Unmasked` in Kubernetes, however, the configuration is hard on Kubernetes, as it has to be used in conjunction with `hostUsers: false`. - Remove `--device /dev/fuse`, as fuse-overlayfs is no longer used typically. - Use the new Kubernetes struct for AppArmor - Add a hint about `kernel.apparmor_restrict_unprivileged_userns` - Remove `$` from command snippets for ease of copypasting - Make `job.*.yaml` more practical - Add `*.userns.yaml`. Needs `UserNamespaceSupport` feature gate to be enabled. Signed-off-by: Akihiro Suda --- docs/rootless.md | 111 ++++++++++++------ examples/kubernetes/README.md | 56 +++++---- .../deployment+service.rootless.yaml | 5 +- .../kubernetes/deployment+service.userns.yaml | 77 ++++++++++++ examples/kubernetes/job.privileged.yaml | 4 +- examples/kubernetes/job.rootless.yaml | 10 +- examples/kubernetes/job.userns.yaml | 47 ++++++++ examples/kubernetes/pod.rootless.yaml | 5 +- examples/kubernetes/pod.userns.yaml | 29 +++++ examples/kubernetes/statefulset.rootless.yaml | 5 +- examples/kubernetes/statefulset.userns.yaml | 42 +++++++ 11 files changed, 317 insertions(+), 74 deletions(-) create mode 100644 examples/kubernetes/deployment+service.userns.yaml create mode 100644 examples/kubernetes/job.userns.yaml create mode 100644 examples/kubernetes/pod.userns.yaml create mode 100644 examples/kubernetes/statefulset.userns.yaml diff --git a/docs/rootless.md b/docs/rootless.md index b2a05883e20e..aafbdf52d1ad 100644 --- a/docs/rootless.md +++ b/docs/rootless.md @@ -12,18 +12,19 @@ Rootless mode allows running BuildKit daemon as a non-root user. [RootlessKit](https://github.com/rootless-containers/rootlesskit/) needs to be installed. -```console -$ rootlesskit buildkitd +```bash +rootlesskit buildkitd ``` -```console -$ buildctl --addr unix:///run/user/$UID/buildkit/buildkitd.sock build ... +```bash +buildctl --addr unix:///run/user/$UID/buildkit/buildkitd.sock build ... ``` -To isolate BuildKit daemon's network namespace from the host (recommended): -```console -$ rootlesskit --net=slirp4netns --copy-up=/etc --disable-host-loopback buildkitd -``` +> [!TIP] +> To isolate BuildKit daemon's network namespace from the host (recommended): +> ```bash +> rootlesskit --net=slirp4netns --copy-up=/etc --disable-host-loopback buildkitd +> ``` ## Running BuildKit in Rootless mode (containerd worker) @@ -31,15 +32,28 @@ $ rootlesskit --net=slirp4netns --copy-up=/etc --disable-host-loopback buildkitd Run containerd in rootless mode using rootlesskit following [containerd's document](https://github.com/containerd/containerd/blob/main/docs/rootless.md). +```bash +containerd-rootless.sh + +CONTAINERD_NAMESPACE=default containerd-rootless-setuptool.sh install-buildkit-containerd ``` -$ containerd-rootless.sh -``` -Then let buildkitd join the same namespace as containerd. +
+Advanced guide + +

+ +Alternatively, you can specify the full command line flags as follows: +```bash +containerd-rootless.sh --config /path/to/config.toml + +containerd-rootless-setuptool.sh nsenter -- buildkitd --oci-worker=false --containerd-worker=true ``` -$ containerd-rootless-setuptool.sh nsenter -- buildkitd --oci-worker=false --containerd-worker=true --containerd-worker-snapshotter=native -``` + +

+ +
## Containerized deployment @@ -48,36 +62,45 @@ See [`../examples/kubernetes`](../examples/kubernetes). ### Docker -```console -$ docker run \ +```bash +docker run \ --name buildkitd \ -d \ --security-opt seccomp=unconfined \ --security-opt apparmor=unconfined \ - --device /dev/fuse \ - moby/buildkit:rootless --oci-worker-no-process-sandbox -$ buildctl --addr docker-container://buildkitd build ... -``` + --security-opt systempaths=unconfined \ + moby/buildkit:rootless -If you don't mind using `--privileged` (almost safe for rootless), the `docker run` flags can be shorten as follows: - -```console -$ docker run --name buildkitd -d --privileged moby/buildkit:rootless +buildctl --addr docker-container://buildkitd build ... ``` -#### About `--device /dev/fuse` -Adding `--device /dev/fuse` to the `docker run` arguments is required only if you want to use `fuse-overlayfs` snapshotter. +> [!TIP] +> If you don't mind using `--privileged` (almost safe for rootless), the `docker run` flags can be shorten as follows: +> +> ```bash +> docker run --name buildkitd -d --privileged moby/buildkit:rootless +> ``` -#### About `--oci-worker-no-process-sandbox` +Justification of the `--security-opt` flags: -By adding `--oci-worker-no-process-sandbox` to the `buildkitd` arguments, BuildKit can be executed in a container without adding `--privileged` to `docker run` arguments. -However, you still need to pass `--security-opt seccomp=unconfined --security-opt apparmor=unconfined` to `docker run`. +* `seccomp=unconfined`: For allowing several syscalls such as `unshare` (used by runc) and `mount` (used by snapshotters, etc). -Note that `--oci-worker-no-process-sandbox` allows build executor containers to `kill` (and potentially `ptrace` depending on the seccomp configuration) an arbitrary process in the BuildKit daemon container. +* `apparmor=unconfined`: For allowing mounting filesystems, etc. + This flag is not needed when the host operating system does not use AppArmor. -To allow running rootless `buildkitd` without `--oci-worker-no-process-sandbox`, run `docker run` with `--security-opt systempaths=unconfined`. (For Kubernetes, set `securityContext.procMount` to `Unmasked`.) +* `systempaths=unconfined`: For disabling the masks for the `/proc` mount in the container, so that each of `ExecOp` + (corresponds to a `RUN` instruction in Dockerfile) can have a dedicated `/proc` filesystem. + `systempaths=unconfined` potentially allows reading and writing dangerous kernel files from a container, but it is safe when you are running `buildkitd` as non-root. -The `--security-opt systempaths=unconfined` flag disables the masks for the `/proc` mount in the container and potentially allows reading and writing dangerous kernel files, but it is safe when you are running `buildkitd` as non-root. +> [!TIP] +> Instead of `--security-opt systempaths=unconfined`, `buildkitd` can be also executed with `--oci-worker-no-process-sandbox` (flag of `buildkitd`, not `docker`) +> to avoid creating a new PID namespace and mounting a new `/proc` for it. +> +> Using `--oci-worker-no-process-sandbox` is discouraged, as it cannot terminate processes that did not exit during an `ExecOp`. +> Also, `--oci-worker-no-process-sandbox` allows `ExecOp` containers to `kill` (and potentially `ptrace` depending on the seccomp configuration) an arbitrary process in the BuildKit daemon container. +> +> Despite these caveats, the [Kubernetes examples](../examples/kubernetes) uses `--oci-worker-no-process-sandbox`, as Kubernetes lacks the equivalent of `systempaths=unconfined`. +> (`securityContext.procMount=Unmasked` is similar, but different in the sense that it depends on `hostUsers: false`) ### Change UID/GID @@ -90,7 +113,7 @@ Actual ID (shown in the host and the BuildKit daemon container)| Mapped ID (show ... | ... 165535 | 65536 -``` +```console $ docker exec buildkitd id uid=1000(user) gid=1000(user) $ docker exec buildkitd ps aux @@ -99,15 +122,16 @@ PID USER TIME COMMAND 13 user 0:00 /proc/self/exe buildkitd --addr tcp://0.0.0.0:1234 21 user 0:00 buildkitd --addr tcp://0.0.0.0:1234 29 user 0:00 ps aux + $ docker exec cat /etc/subuid user:100000:65536 ``` To change the UID/GID configuration, you need to modify and build the BuildKit image manually. -``` -$ vi Dockerfile -$ make images -$ docker run ... moby/buildkit:local-rootless ... +```bash +vi Dockerfile +make images +docker run ... moby/buildkit:local-rootless ... ``` ## Troubleshooting @@ -120,7 +144,9 @@ $ rootlesskit buildkitd --oci-worker-snapshotter=fuse-overlayfs ``` ### Error related to `fuse-overlayfs` -Try running `buildkitd` with `--oci-worker-snapshotter=native`: +Run `docker run` with `--device /dev/fuse`. + +Also try running `buildkitd` with `--oci-worker-snapshotter=native`: ```console $ rootlesskit buildkitd --oci-worker-snapshotter=native @@ -137,12 +163,19 @@ Run `sysctl -w user.max_user_namespaces=N` (N=positive integer, like 63359) on t See [`../examples/kubernetes/sysctl-userns.privileged.yaml`](../examples/kubernetes/sysctl-userns.privileged.yaml). +### Error `fork/exec /proc/self/exe: permission denied` with `This error might have happened because /proc/sys/kernel/apparmor_restrict_unprivileged_userns is set to 1` +Add `kernel.apparmor_restrict_unprivileged_userns=0` to `/etc/sysctl.conf` (or `/etc/sysctl.d`) and run `sudo sysctl -p`. + ### Error `mount proc:/proc (via /proc/self/fd/6), flags: 0xe: operation not permitted` -This error is known to happen when BuildKit is executed in a container without the `--oci-worker-no-sandbox` flag. -Make sure that `--oci-worker-no-process-sandbox` is specified (See [below](#docker)). +This error is known to happen when BuildKit is executed in a container without the `--security-opt systempaths=unconfined` flag. +Make sure to specify it (See [above](#docker)). ## Distribution-specific hint Using Ubuntu kernel is recommended. + +### Ubuntu, 24.04 or later +Add `kernel.apparmor_restrict_unprivileged_userns=0` to `/etc/sysctl.conf` (or `/etc/sysctl.d`) and run `sudo sysctl -p`. + ### Container-Optimized OS from Google Make sure to have an `emptyDir` volume below: ```yaml diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md index c8973dc5645d..4d263d16c2eb 100644 --- a/examples/kubernetes/README.md +++ b/examples/kubernetes/README.md @@ -6,16 +6,26 @@ This directory contains Kubernetes manifests for `Pod`, `Deployment` (with `Serv * `StateFulset`: good for client-side load balancing, without registry-side cache * `Job`: good if you don't want to have daemon pods -Using Rootless mode (`*.rootless.yaml`) is recommended because Rootless mode image is executed as non-root user (UID 1000) and doesn't need `securityContext.privileged`. -See [`../../docs/rootless.md`](../../docs/rootless.md). +## Variants -See also ["Building Images Efficiently And Securely On Kubernetes With BuildKit" (KubeCon EU 2019)](https://kccnceu19.sched.com/event/MPX5). +- `*.privileged.yaml`: Launches the Pod as the fully privileged root user. +- `*.rootless.yaml`: Launches the Pod as a non-root user, whose UID is 1000. +- `*.userns.yaml`: Launches the Pod as a non-root user. The UID is determined by kubelet. + Needs kubelet and kube-apiserver to be reconfigured to enable the + [`UserNamespacesSupport`](https://kubernetes.io/docs/tasks/configure-pod-container/user-namespaces/) feature gate. + +It is recommended to use `*.rootless.yaml` to minimize the chance of container breakout attacks. + +See also: +- [`../../docs/rootless.md`](../../docs/rootless.md). +- ["Building Images Efficiently And Securely On Kubernetes With BuildKit" (KubeCon EU 2019)](https://kccnceu19.sched.com/event/MPX5). ## `Pod` -```console -$ kubectl apply -f pod.rootless.yaml -$ buildctl \ +```bash +kubectl apply -f pod.rootless.yaml + +buildctl \ --addr kube-pod://buildkitd \ build --frontend dockerfile.v0 --local context=/path/to/dir --local dockerfile=/path/to/dir ``` @@ -29,25 +39,27 @@ If rootless mode doesn't work, try `pod.privileged.yaml`. Setting up mTLS is highly recommended. `./create-certs.sh SAN [SAN...]` can be used for creating certificates. -```console -$ ./create-certs.sh 127.0.0.1 +```bash +./create-certs.sh 127.0.0.1 ``` The daemon certificates is created as `Secret` manifest named `buildkit-daemon-certs`. -```console -$ kubectl apply -f .certs/buildkit-daemon-certs.yaml +```bash +kubectl apply -f .certs/buildkit-daemon-certs.yaml ``` Apply the `Deployment` and `Service` manifest: -```console -$ kubectl apply -f deployment+service.rootless.yaml -$ kubectl scale --replicas=10 deployment/buildkitd +```bash +kubectl apply -f deployment+service.rootless.yaml + +kubectl scale --replicas=10 deployment/buildkitd ``` Run `buildctl` with TLS client certificates: -```console -$ kubectl port-forward service/buildkitd 1234 -$ buildctl \ +```bash +kubectl port-forward service/buildkitd 1234 + +buildctl \ --addr tcp://127.0.0.1:1234 \ --tlscacert .certs/client/ca.pem \ --tlscert .certs/client/cert.pem \ @@ -58,10 +70,10 @@ $ buildctl \ ## `StatefulSet` `StatefulSet` is useful for consistent hash mode. -```console -$ kubectl apply -f statefulset.rootless.yaml -$ kubectl scale --replicas=10 statefulset/buildkitd -$ buildctl \ +```bash +kubectl apply -f statefulset.rootless.yaml +kubectl scale --replicas=10 statefulset/buildkitd +buildctl \ --addr kube-pod://buildkitd-4 \ build --frontend dockerfile.v0 --local context=/path/to/dir --local dockerfile=/path/to/dir ``` @@ -70,8 +82,8 @@ See [`./consistenthash`](./consistenthash) for how to use consistent hashing. ## `Job` -```console -$ kubectl apply -f job.rootless.yaml +```bash +kubectl apply -f job.rootless.yaml ``` To push the image to the registry, you also need to mount `~/.docker/config.json` diff --git a/examples/kubernetes/deployment+service.rootless.yaml b/examples/kubernetes/deployment+service.rootless.yaml index 0b554096fde6..c82ff9820a3f 100644 --- a/examples/kubernetes/deployment+service.rootless.yaml +++ b/examples/kubernetes/deployment+service.rootless.yaml @@ -13,8 +13,6 @@ spec: metadata: labels: app: buildkitd - annotations: - container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined # see buildkit/docs/rootless.md for caveats of rootless mode spec: containers: @@ -54,6 +52,9 @@ spec: # Needs Kubernetes >= 1.19 seccompProfile: type: Unconfined + # Needs Kubernetes >= 1.30 + appArmorProfile: + type: Unconfined # To change UID/GID, you need to rebuild the image runAsUser: 1000 runAsGroup: 1000 diff --git a/examples/kubernetes/deployment+service.userns.yaml b/examples/kubernetes/deployment+service.userns.yaml new file mode 100644 index 000000000000..acf19937ef4d --- /dev/null +++ b/examples/kubernetes/deployment+service.userns.yaml @@ -0,0 +1,77 @@ +# Depends on feature gate UserNamespacesSupport +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: buildkitd + name: buildkitd +spec: + replicas: 1 + selector: + matchLabels: + app: buildkitd + template: + metadata: + labels: + app: buildkitd + spec: + hostUsers: false + containers: + - name: buildkitd + image: moby/buildkit:master + args: + - --addr + - unix:///run/buildkit/buildkitd.sock + - --addr + - tcp://0.0.0.0:1234 + - --tlscacert + - /certs/ca.pem + - --tlscert + - /certs/cert.pem + - --tlskey + - /certs/key.pem + # the probe below will only work after Release v0.6.3 + readinessProbe: + exec: + command: + - buildctl + - debug + - workers + initialDelaySeconds: 5 + periodSeconds: 30 + # the probe below will only work after Release v0.6.3 + livenessProbe: + exec: + command: + - buildctl + - debug + - workers + initialDelaySeconds: 5 + periodSeconds: 30 + securityContext: + # Not really privileged + privileged: true + ports: + - containerPort: 1234 + volumeMounts: + - name: certs + readOnly: true + mountPath: /certs + volumes: + # buildkit-daemon-certs must contain ca.pem, cert.pem, and key.pem + - name: certs + secret: + secretName: buildkit-daemon-certs +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: buildkitd + name: buildkitd +spec: + ports: + - port: 1234 + protocol: TCP + selector: + app: buildkitd diff --git a/examples/kubernetes/job.privileged.yaml b/examples/kubernetes/job.privileged.yaml index 352180efab57..472fc0d3516f 100644 --- a/examples/kubernetes/job.privileged.yaml +++ b/examples/kubernetes/job.privileged.yaml @@ -8,11 +8,11 @@ spec: restartPolicy: Never initContainers: - name: prepare - image: alpine:3.10 + image: busybox command: - sh - -c - - "echo FROM hello-world > /workspace/Dockerfile" + - "echo -e 'FROM alpine\nRUN apk add gcc\n' > /workspace/Dockerfile" volumeMounts: - name: workspace mountPath: /workspace diff --git a/examples/kubernetes/job.rootless.yaml b/examples/kubernetes/job.rootless.yaml index 06e608c6ab35..a3904f8d659e 100644 --- a/examples/kubernetes/job.rootless.yaml +++ b/examples/kubernetes/job.rootless.yaml @@ -4,19 +4,16 @@ metadata: name: buildkit spec: template: - metadata: - annotations: - container.apparmor.security.beta.kubernetes.io/buildkit: unconfined # see buildkit/docs/rootless.md for caveats of rootless mode spec: restartPolicy: Never initContainers: - name: prepare - image: alpine:3.10 + image: busybox command: - sh - -c - - "echo FROM hello-world > /workspace/Dockerfile" + - "echo -e 'FROM alpine\nRUN apk add gcc\n' > /workspace/Dockerfile" securityContext: runAsUser: 1000 runAsGroup: 1000 @@ -45,6 +42,9 @@ spec: # Needs Kubernetes >= 1.19 seccompProfile: type: Unconfined + # Needs Kubernetes >= 1.30 + appArmorProfile: + type: Unconfined # To change UID/GID, you need to rebuild the image runAsUser: 1000 runAsGroup: 1000 diff --git a/examples/kubernetes/job.userns.yaml b/examples/kubernetes/job.userns.yaml new file mode 100644 index 000000000000..9305bce14073 --- /dev/null +++ b/examples/kubernetes/job.userns.yaml @@ -0,0 +1,47 @@ +# Depends on feature gate UserNamespacesSupport +apiVersion: batch/v1 +kind: Job +metadata: + name: buildkit +spec: + template: + spec: + hostUsers: false + restartPolicy: Never + initContainers: + - name: prepare + image: busybox + command: + - sh + - -c + - "echo -e 'FROM alpine\nRUN apk add gcc\n' > /workspace/Dockerfile" + volumeMounts: + - name: workspace + mountPath: /workspace + containers: + - name: buildkit + image: moby/buildkit:master + command: + - buildctl-daemonless.sh + args: + - build + - --frontend + - dockerfile.v0 + - --local + - context=/workspace + - --local + - dockerfile=/workspace + # To push the image to a registry, add + # `--output type=image,name=docker.io/username/image,push=true` + securityContext: + # Not really privileged + privileged: true + volumeMounts: + - name: workspace + readOnly: true + mountPath: /workspace + # To push the image, you also need to create `~/.docker/config.json` secret + # and set $DOCKER_CONFIG to `/path/to/.docker` directory. + volumes: + - name: workspace + emptyDir: {} diff --git a/examples/kubernetes/pod.rootless.yaml b/examples/kubernetes/pod.rootless.yaml index 130ea43633fe..4f9864594b37 100644 --- a/examples/kubernetes/pod.rootless.yaml +++ b/examples/kubernetes/pod.rootless.yaml @@ -2,8 +2,6 @@ apiVersion: v1 kind: Pod metadata: name: buildkitd - annotations: - container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined # see buildkit/docs/rootless.md for caveats of rootless mode spec: containers: @@ -31,6 +29,9 @@ spec: # Needs Kubernetes >= 1.19 seccompProfile: type: Unconfined + # Needs Kubernetes >= 1.30 + appArmorProfile: + type: Unconfined # To change UID/GID, you need to rebuild the image runAsUser: 1000 runAsGroup: 1000 diff --git a/examples/kubernetes/pod.userns.yaml b/examples/kubernetes/pod.userns.yaml new file mode 100644 index 000000000000..085c3cde0ee9 --- /dev/null +++ b/examples/kubernetes/pod.userns.yaml @@ -0,0 +1,29 @@ +# Depends on feature gate UserNamespacesSupport +apiVersion: v1 +kind: Pod +metadata: + name: buildkitd +spec: + hostUsers: false + containers: + - name: buildkitd + image: moby/buildkit:master + readinessProbe: + exec: + command: + - buildctl + - debug + - workers + initialDelaySeconds: 5 + periodSeconds: 30 + livenessProbe: + exec: + command: + - buildctl + - debug + - workers + initialDelaySeconds: 5 + periodSeconds: 30 + securityContext: + # Not really privileged + privileged: true diff --git a/examples/kubernetes/statefulset.rootless.yaml b/examples/kubernetes/statefulset.rootless.yaml index 0533d2a1004f..caf7dde3cc2b 100644 --- a/examples/kubernetes/statefulset.rootless.yaml +++ b/examples/kubernetes/statefulset.rootless.yaml @@ -15,8 +15,6 @@ spec: metadata: labels: app: buildkitd - annotations: - container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined # see buildkit/docs/rootless.md for caveats of rootless mode spec: containers: @@ -44,6 +42,9 @@ spec: # Needs Kubernetes >= 1.19 seccompProfile: type: Unconfined + # Needs Kubernetes >= 1.30 + appArmorProfile: + type: Unconfined # To change UID/GID, you need to rebuild the image runAsUser: 1000 runAsGroup: 1000 diff --git a/examples/kubernetes/statefulset.userns.yaml b/examples/kubernetes/statefulset.userns.yaml new file mode 100644 index 000000000000..98af0aad9ef3 --- /dev/null +++ b/examples/kubernetes/statefulset.userns.yaml @@ -0,0 +1,42 @@ +# Depends on feature gate UserNamespacesSupport +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: buildkitd + name: buildkitd +spec: + serviceName: buildkitd + podManagementPolicy: Parallel + replicas: 1 + selector: + matchLabels: + app: buildkitd + template: + metadata: + labels: + app: buildkitd + spec: + hostUsers: false + containers: + - name: buildkitd + image: moby/buildkit:master + readinessProbe: + exec: + command: + - buildctl + - debug + - workers + initialDelaySeconds: 5 + periodSeconds: 30 + livenessProbe: + exec: + command: + - buildctl + - debug + - workers + initialDelaySeconds: 5 + periodSeconds: 30 + securityContext: + # Not really privileged + privileged: true From 2c40436cbb1428aa345847de420174ec1b18e323 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 24 Feb 2025 20:20:10 -0800 Subject: [PATCH 15/23] dockerfile: normalize platform in image config Base image may use unnormalized platform so if platform is inherited normalize needs to be called again. Signed-off-by: Tonis Tiigi --- frontend/dockerfile/dockerfile2llb/convert.go | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index f44866839341..77c91c15d336 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -815,6 +815,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS target.image.OSFeatures = append([]string{}, platformOpt.targetPlatform.OSFeatures...) } } + target.image.Platform = platforms.Normalize(target.image.Platform) return target, nil } From cc210c8833940ce33eac059fbd04f5de3c3f898f Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Fri, 28 Feb 2025 16:06:52 -0800 Subject: [PATCH 16/23] vendor: update containerd to v2.0.3 Brings in the gRPC message size fix for writing SBOMs. Signed-off-by: Tonis Tiigi --- go.mod | 2 +- go.sum | 4 +- .../v2/core/content/proxy/content_writer.go | 45 ++++++++++++------- .../containerd/v2/pkg/oci/spec_opts.go | 41 +++-------------- .../containerd/v2/version/version.go | 2 +- vendor/modules.txt | 2 +- 6 files changed, 39 insertions(+), 57 deletions(-) diff --git a/go.mod b/go.mod index 166697f4ac4e..dd696567a3cb 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 github.com/containerd/console v1.0.4 github.com/containerd/containerd/api v1.8.0 - github.com/containerd/containerd/v2 v2.0.2 + github.com/containerd/containerd/v2 v2.0.3 github.com/containerd/continuity v0.4.5 github.com/containerd/errdefs v1.0.0 github.com/containerd/fuse-overlayfs-snapshotter/v2 v2.1.1 diff --git a/go.sum b/go.sum index fa32deebbfc0..fdca30d64082 100644 --- a/go.sum +++ b/go.sum @@ -97,8 +97,8 @@ github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc= -github.com/containerd/containerd/v2 v2.0.2 h1:GmH/tRBlTvrXOLwSpWE2vNAm8+MqI6nmxKpKBNKY8Wc= -github.com/containerd/containerd/v2 v2.0.2/go.mod h1:wIqEvQ/6cyPFUGJ5yMFanspPabMLor+bF865OHvNTTI= +github.com/containerd/containerd/v2 v2.0.3 h1:zBKgwgZsuu+LPCMzCLgA4sC4MiZzZ59ZT31XkmiISQM= +github.com/containerd/containerd/v2 v2.0.3/go.mod h1:5j9QUUaV/cy9ZeAx4S+8n9ffpf+iYnEj4jiExgcbuLY= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= diff --git a/vendor/github.com/containerd/containerd/v2/core/content/proxy/content_writer.go b/vendor/github.com/containerd/containerd/v2/core/content/proxy/content_writer.go index 214a0a335172..98818f234ab0 100644 --- a/vendor/github.com/containerd/containerd/v2/core/content/proxy/content_writer.go +++ b/vendor/github.com/containerd/containerd/v2/core/content/proxy/content_writer.go @@ -26,6 +26,7 @@ import ( digest "github.com/opencontainers/go-digest" "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/defaults" "github.com/containerd/containerd/v2/pkg/protobuf" ) @@ -76,27 +77,37 @@ func (rw *remoteWriter) Digest() digest.Digest { } func (rw *remoteWriter) Write(p []byte) (n int, err error) { - offset := rw.offset + const maxBufferSize = defaults.DefaultMaxSendMsgSize >> 1 + for i := 0; i < len(p); i += maxBufferSize { + offset := rw.offset - resp, err := rw.send(&contentapi.WriteContentRequest{ - Action: contentapi.WriteAction_WRITE, - Offset: offset, - Data: p, - }) - if err != nil { - return 0, fmt.Errorf("failed to send write: %w", errgrpc.ToNative(err)) - } + end := i + maxBufferSize + if end > len(p) { + end = len(p) + } + data := p[i:end] + + resp, err := rw.send(&contentapi.WriteContentRequest{ + Action: contentapi.WriteAction_WRITE, + Offset: offset, + Data: data, + }) + if err != nil { + return 0, fmt.Errorf("failed to send write: %w", errgrpc.ToNative(err)) + } - n = int(resp.Offset - offset) - if n < len(p) { - err = io.ErrShortWrite - } + written := int(resp.Offset - offset) + rw.offset += int64(written) + if resp.Digest != "" { + rw.digest = digest.Digest(resp.Digest) + } + n += written - rw.offset += int64(n) - if resp.Digest != "" { - rw.digest = digest.Digest(resp.Digest) + if written < len(data) { + return n, io.ErrShortWrite + } } - return + return n, nil } func (rw *remoteWriter) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) (err error) { diff --git a/vendor/github.com/containerd/containerd/v2/pkg/oci/spec_opts.go b/vendor/github.com/containerd/containerd/v2/pkg/oci/spec_opts.go index 5101c63bb638..3b85d764ae10 100644 --- a/vendor/github.com/containerd/containerd/v2/pkg/oci/spec_opts.go +++ b/vendor/github.com/containerd/containerd/v2/pkg/oci/spec_opts.go @@ -28,18 +28,17 @@ import ( "strconv" "strings" - "github.com/containerd/containerd/v2/core/containers" - "github.com/containerd/containerd/v2/core/content" - "github.com/containerd/containerd/v2/core/images" - "github.com/containerd/containerd/v2/core/mount" - "github.com/containerd/containerd/v2/pkg/namespaces" "github.com/containerd/continuity/fs" - "github.com/containerd/log" "github.com/containerd/platforms" "github.com/moby/sys/user" v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-spec/specs-go" - "tags.cncf.io/container-device-interface/pkg/cdi" + + "github.com/containerd/containerd/v2/core/containers" + "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/images" + "github.com/containerd/containerd/v2/core/mount" + "github.com/containerd/containerd/v2/pkg/namespaces" ) // SpecOpts sets spec specific information to a newly generated OCI spec @@ -1643,31 +1642,3 @@ func WithWindowsNetworkNamespace(ns string) SpecOpts { return nil } } - -// WithCDIDevices injects the requested CDI devices into the OCI specification. -func WithCDIDevices(devices ...string) SpecOpts { - return func(ctx context.Context, _ Client, c *containers.Container, s *Spec) error { - if len(devices) == 0 { - return nil - } - - if err := cdi.Refresh(); err != nil { - // We don't consider registry refresh failure a fatal error. - // For instance, a dynamically generated invalid CDI Spec file for - // any particular vendor shouldn't prevent injection of devices of - // different vendors. CDI itself knows better and it will fail the - // injection if necessary. - log.G(ctx).Warnf("CDI registry refresh failed: %v", err) - } - - if _, err := cdi.InjectDevices(s, devices...); err != nil { - return fmt.Errorf("CDI device injection failed: %w", err) - } - - // One crucial thing to keep in mind is that CDI device injection - // might add OCI Spec environment variables, hooks, and mounts as - // well. Therefore it is important that none of the corresponding - // OCI Spec fields are reset up in the call stack once we return. - return nil - } -} diff --git a/vendor/github.com/containerd/containerd/v2/version/version.go b/vendor/github.com/containerd/containerd/v2/version/version.go index 28f46ed09cab..d0749a6cfcd3 100644 --- a/vendor/github.com/containerd/containerd/v2/version/version.go +++ b/vendor/github.com/containerd/containerd/v2/version/version.go @@ -23,7 +23,7 @@ var ( Package = "github.com/containerd/containerd/v2" // Version holds the complete version number. Filled in at linking time. - Version = "2.0.2+unknown" + Version = "2.0.3+unknown" // Revision is filled with the VCS (e.g. git) revision being used to build // the program at linking time. diff --git a/vendor/modules.txt b/vendor/modules.txt index e95c71415741..7dfe5e911ffd 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -301,7 +301,7 @@ github.com/containerd/containerd/api/types/runc/options github.com/containerd/containerd/api/types/runtimeoptions/v1 github.com/containerd/containerd/api/types/task github.com/containerd/containerd/api/types/transfer -# github.com/containerd/containerd/v2 v2.0.2 +# github.com/containerd/containerd/v2 v2.0.3 ## explicit; go 1.22.0 github.com/containerd/containerd/v2/client github.com/containerd/containerd/v2/core/containers From 15c25d430beb71420018e576793c0ee2d11a1e0f Mon Sep 17 00:00:00 2001 From: Pudong Zheng Date: Wed, 20 Sep 2023 06:14:29 +0000 Subject: [PATCH 17/23] add bob proxy as only mirror --- util/resolver/resolver.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/util/resolver/resolver.go b/util/resolver/resolver.go index 8308299c109f..6dca73461116 100644 --- a/util/resolver/resolver.go +++ b/util/resolver/resolver.go @@ -122,12 +122,14 @@ func loadTLSConfig(c config.RegistryConfig) (*tls.Config, error) { // NewRegistryConfig converts registry config to docker.RegistryHosts callback func NewRegistryConfig(m map[string]config.RegistryConfig) docker.RegistryHosts { + mirrorHost := "localhost:8080" return docker.Registries( func(host string) ([]docker.RegistryHost, error) { c, ok := m[host] if !ok { - return nil, nil + c = config.RegistryConfig{} } + c.Mirrors = []string{mirrorHost} var out []docker.RegistryHost From 12fd7ed0ee5c93890e43e0e20af6cb01ba3737af Mon Sep 17 00:00:00 2001 From: Pudong Zheng Date: Wed, 20 Sep 2023 06:14:38 +0000 Subject: [PATCH 18/23] ci --- .github/workflows/buildkit.yml | 20 +-- .github/workflows/buildx-image.yml | 81 --------- .github/workflows/dockerd.yml | 134 -------------- .github/workflows/docs-upstream.yml | 35 ---- .github/workflows/frontend.yml | 186 -------------------- .github/workflows/test-os.yml | 261 ---------------------------- .github/workflows/validate.yml | 82 --------- 7 files changed, 7 insertions(+), 792 deletions(-) delete mode 100644 .github/workflows/buildx-image.yml delete mode 100644 .github/workflows/dockerd.yml delete mode 100644 .github/workflows/docs-upstream.yml delete mode 100644 .github/workflows/frontend.yml delete mode 100644 .github/workflows/test-os.yml delete mode 100644 .github/workflows/validate.yml diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index c17f80e8ee70..c8e25c4d6d50 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -5,8 +5,6 @@ concurrency: cancel-in-progress: true on: - schedule: - - cron: '0 10 * * *' workflow_dispatch: push: branches: @@ -24,9 +22,8 @@ env: GO_VERSION: "1.23" SETUP_BUILDX_VERSION: "edge" SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" - SCOUT_VERSION: "1.13.0" - IMAGE_NAME: "moby/buildkit" - PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le,linux/riscv64" + IMAGE_NAME: "ghcr.io/gitpod-io/buildkit" + PLATFORMS: "linux/amd64" DESTDIR: "./bin" jobs: @@ -63,8 +60,7 @@ jobs: fi echo "tag=${TAG}" >>${GITHUB_OUTPUT} echo "push=${PUSH}" >>${GITHUB_OUTPUT} - platforms=$(docker buildx bake release --print | jq -cr '.target."release".platforms') - echo "platforms=$platforms" >>${GITHUB_OUTPUT} + echo ::set-output name=platforms::'["linux/amd64"]' binaries: runs-on: ubuntu-24.04 @@ -180,13 +176,11 @@ jobs: runs-on: ubuntu-24.04 needs: - prepare - - test strategy: fail-fast: false matrix: target-stage: - '' - - rootless steps: - name: Checkout @@ -205,12 +199,13 @@ jobs: driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} buildkitd-flags: --debug - - name: Login to DockerHub + name: Login to GitHub Container Registry if: needs.prepare.outputs.push == 'push' uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Build ${{ needs.prepare.outputs.tag }} run: | @@ -268,7 +263,6 @@ jobs: runs-on: ubuntu-24.04 needs: - prepare - - test - binaries - image steps: diff --git a/.github/workflows/buildx-image.yml b/.github/workflows/buildx-image.yml deleted file mode 100644 index f3f00e3b9f81..000000000000 --- a/.github/workflows/buildx-image.yml +++ /dev/null @@ -1,81 +0,0 @@ -# source latest -# dest buildx-stable-1 -# result moby/buildkit:latest > moby/buildkit:buildx-stable-1 -# moby/buildkit:rootless > moby/buildkit:buildx-stable-1-rootless -# -# source v0.8.1 -# dest buildx-stable-1 -# result moby/buildkit:v0.8.1 > moby/buildkit:buildx-stable-1 -# moby/buildkit:v0.8.1-rootless > moby/buildkit:buildx-stable-1-rootless -name: buildx-image - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - inputs: - source-tag: - description: 'BuildKit source Docker tag' - required: true - default: 'latest' - dest-tag: - description: 'Default BuildKit Docker tag for buildx' - required: true - default: 'buildx-stable-1' - dry-run: - description: 'Dry run' - required: false - default: 'true' - -env: - SETUP_BUILDX_VERSION: "edge" - REPO_SLUG_TARGET: "moby/buildkit" - -jobs: - create: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - flavor: - - '' - - 'rootless' - steps: - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: ${{ env.SETUP_BUILDX_VERSION }} - buildkitd-flags: --debug - - - name: Login to DockerHub - if: github.event.inputs.dry-run != 'true' - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Create - run: | - DRYRUN_FLAG="" - if [ "${{ github.event.inputs.dry-run }}" = "true" ]; then - DRYRUN_FLAG="--dry-run" - fi - - SOURCE_TAG="${{ github.event.inputs.source-tag }}" - DEST_TAG="${{ github.event.inputs.dest-tag }}" - if [ "${{ matrix.flavor }}" != "" ]; then - if [ "$SOURCE_TAG" = "latest" ]; then - SOURCE_TAG="${{ matrix.flavor }}" - else - SOURCE_TAG="${SOURCE_TAG}-${{ matrix.flavor }}" - fi - DEST_TAG="${DEST_TAG}-${{ matrix.flavor }}" - fi - - set -x - docker buildx imagetools create ${DRYRUN_FLAG} --tag \ - "${{ env.REPO_SLUG_TARGET }}:${DEST_TAG}" \ - "${{ env.REPO_SLUG_TARGET }}:${SOURCE_TAG}" diff --git a/.github/workflows/dockerd.yml b/.github/workflows/dockerd.yml deleted file mode 100644 index b323ac404944..000000000000 --- a/.github/workflows/dockerd.yml +++ /dev/null @@ -1,134 +0,0 @@ -name: dockerd - -on: - # TODO: add event to build on command in PR (e.g., /test-dockerd) - workflow_dispatch: - inputs: - version: - description: 'Docker version' - required: true - default: '25.0.2' - -env: - SETUP_BUILDX_VERSION: "edge" - SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" - TESTFLAGS: "-v --parallel=1 --timeout=30m" - -jobs: - prepare: - runs-on: ubuntu-24.04 - steps: - - - name: Prepare - uses: actions/github-script@v7 - with: - script: | - const version = `${{ inputs.version }}` || '25.0.2'; - let build = 'true'; - try { - new URL(version); - } catch (e) { - build = 'false'; - } - core.exportVariable('DOCKER_VERSION', version); - core.exportVariable('DOCKER_BUILD', build); - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: ${{ env.SETUP_BUILDX_VERSION }} - driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} - buildkitd-flags: --debug - - - name: Build - if: ${{ env.DOCKER_BUILD == 'true' }} - uses: docker/build-push-action@v6 - with: - context: ${{ env.DOCKER_VERSION }} - target: binary - outputs: /tmp/moby - - - name: Rename binary - if: ${{ env.DOCKER_BUILD == 'true' }} - run: | - if [ -L "/tmp/moby/binary-daemon/dockerd" ]; then - mv -f $(readlink /tmp/moby/binary-daemon/dockerd) /tmp/moby/dockerd - fi - - - name: Download - if: ${{ env.DOCKER_BUILD != 'true' }} - run: | - mkdir -p /tmp/moby - cd /tmp/moby - wget -qO- "https://download.docker.com/linux/static/stable/x86_64/docker-${{ env.DOCKER_VERSION }}.tgz" | tar xvz --strip 1 - - - name: Upload dockerd - uses: actions/upload-artifact@v4 - with: - name: dockerd - path: /tmp/moby/dockerd - if-no-files-found: error - retention-days: 1 - - test: - runs-on: ubuntu-24.04 - needs: - - prepare - strategy: - fail-fast: false - matrix: - worker: - - dockerd - - dockerd-containerd - pkg: - - ./client - - ./cmd/buildctl - - ./solver - - ./frontend - - ./frontend/dockerfile - typ: - - integration - include: - - pkg: ./... - skip-integration-tests: 1 - steps: - - - name: Checkout - uses: actions/checkout@v4 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: ${{ env.SETUP_BUILDX_VERSION }} - driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} - buildkitd-flags: --debug - - - name: Download dockerd - uses: actions/download-artifact@v4 - with: - name: dockerd - path: ./build/ - - - name: Fix dockerd perms - run: | - chmod +x ./build/dockerd - - - name: Test - run: | - ./hack/test ${{ matrix.typ }} - env: - TEST_DOCKERD: "1" - TEST_DOCKERD_BINARY: "./build/dockerd" - TESTPKGS: "${{ matrix.pkg }}" - TESTFLAGS: "${{ env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$" - SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}" - CACHE_FROM: "type=gha,scope=build-integration-tests" - BUILDKIT_INTEGRATION_DOCKERD_FLAGS: | - --bip=10.66.66.1/24 - --default-address-pool=base=10.66.66.0/16,size=24 diff --git a/.github/workflows/docs-upstream.yml b/.github/workflows/docs-upstream.yml deleted file mode 100644 index 0c9d18e7d503..000000000000 --- a/.github/workflows/docs-upstream.yml +++ /dev/null @@ -1,35 +0,0 @@ -# this workflow runs the remote validate bake target from docker/docs to check -# if yaml reference docs and markdown files used in this repo are still valid -# https://github.com/docker/docker.github.io/blob/98c7c9535063ae4cd2cd0a31478a21d16d2f07a3/docker-bake.hcl#L34-L36 -name: docs-upstream - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: - - 'master' - - 'v[0-9]*' - paths: - - '.github/workflows/docs-upstream.yml' - - 'docs/buildkitd.toml.md' - - 'docs/attestations/slsa-definitions.md' - - 'docs/attestations/attestation-storage.md' - - 'frontend/dockerfile/docs/reference.md' - - 'frontend/dockerfile/docs/rules/**' - pull_request: - paths: - - '.github/workflows/docs-upstream.yml' - - 'docs/buildkitd.toml.md' - - 'docs/attestations/slsa-definitions.md' - - 'docs/attestations/attestation-storage.md' - - 'frontend/dockerfile/docs/reference.md' - - 'frontend/dockerfile/docs/rules/**' - -jobs: - validate: - uses: docker/docs/.github/workflows/validate-upstream.yml@main - with: - module-name: moby/buildkit diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml deleted file mode 100644 index ff5ffece80fa..000000000000 --- a/.github/workflows/frontend.yml +++ /dev/null @@ -1,186 +0,0 @@ -name: frontend - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - push: - branches: - - 'master' - - 'v[0-9]+.[0-9]+' - tags: - - 'dockerfile/*' - pull_request: - paths-ignore: - - 'README.md' - - 'docs/**' - - 'frontend/dockerfile/docs/**' - -env: - GO_VERSION: "1.23" - SETUP_BUILDX_VERSION: "edge" - SETUP_BUILDKIT_TAG: "moby/buildkit:latest" - SCOUT_VERSION: "1.13.0" - IMAGE_NAME: "docker/dockerfile-upstream" - PLATFORMS: "linux/386,linux/amd64,linux/arm/v7,linux/arm64,linux/mips,linux/mipsle,linux/mips64,linux/mips64le,linux/s390x,linux/ppc64le,linux/riscv64" - -jobs: - test: - uses: ./.github/workflows/.test.yml - secrets: inherit - with: - cache_scope: frontend-integration-tests - pkgs: ./frontend/dockerfile - kinds: | - integration - dockerfile - codecov_flags: dockerfile-frontend - - prepare: - runs-on: ubuntu-24.04 - if: github.event_name != 'schedule' - outputs: - typ: ${{ steps.prep.outputs.typ }} - push: ${{ steps.prep.outputs.push }} - tag: ${{ steps.prep.outputs.tag }} - tags: ${{ steps.prep.outputs.tags }} - steps: - - - name: Prepare - id: prep - run: | - TYP=master - TAG=mainline - PUSH=false - if [[ $GITHUB_REF == refs/tags/dockerfile/* ]]; then - TYP=tag - TAG=${GITHUB_REF#refs/tags/} - PUSH=push - elif [[ $GITHUB_REF == refs/heads/* ]]; then - if [ $GITHUB_REF = "refs/heads/${{ github.event.repository.default_branch }}" ]; then - PUSH=push - fi - fi - if [ "$GITHUB_REPOSITORY" != "moby/buildkit" ]; then - PUSH=false - fi - echo "typ=${TYP}" >>${GITHUB_OUTPUT} - echo "push=${PUSH}" >>${GITHUB_OUTPUT} - echo "tag=${TAG}" >>${GITHUB_OUTPUT} - if [ "${TYP}" = "master" ]; then - echo "tags=$(jq -cn --arg tag "$TAG" '[$tag, "labs"]')" >>${GITHUB_OUTPUT} - else - echo "tags=$(jq -cn --arg tag "$TAG" '[$tag]')" >>${GITHUB_OUTPUT} - fi - - image: - runs-on: ubuntu-24.04 - needs: - - test - - prepare - strategy: - fail-fast: false - matrix: - tag: ${{ fromJson(needs.prepare.outputs.tags) }} - steps: - - - name: Prepare - run: | - if [[ "${{ matrix.tag }}" = "labs" ]] || [[ "${{ matrix.tag }}" == *-labs ]]; then - echo "CACHE_SCOPE=frontend-labs" >>${GITHUB_ENV} - else - echo "CACHE_SCOPE=frontend-mainline" >>${GITHUB_ENV} - fi - - - name: Checkout - uses: actions/checkout@v4 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: ${{ env.SETUP_BUILDX_VERSION }} - driver-opts: image=${{ env.SETUP_BUILDKIT_TAG }} - buildkitd-flags: --debug - - - name: Login to DockerHub - uses: docker/login-action@v3 - if: needs.prepare.outputs.push == 'push' - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build - run: | - ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.prepare.outputs.typ }}" "${{ matrix.tag }}" "$IMAGE_NAME" "${{ needs.prepare.outputs.push }}" - env: - RELEASE: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') }} - CACHE_FROM: type=gha,scope=${{ env.CACHE_SCOPE }} - CACHE_TO: type=gha,scope=${{ env.CACHE_SCOPE }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - scout: - runs-on: ubuntu-24.04 - if: ${{ github.ref == 'refs/heads/master' && github.repository == 'moby/buildkit' }} - permissions: - # required to write sarif report - security-events: write - needs: - - image - strategy: - fail-fast: false - matrix: - tag: - - master - - master-labs - steps: - - - name: Checkout - uses: actions/checkout@v4 - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Scout - id: scout - uses: crazy-max/.github/.github/actions/docker-scout@ccae1c98f1237b5c19e4ef77ace44fa68b3bc7e4 - with: - version: ${{ env.SCOUT_VERSION }} - format: sarif - image: registry://${{ env.IMAGE_NAME }}:${{ matrix.tag }} - - - name: Result output - run: | - jq . ${{ steps.scout.outputs.result-file }} - - - name: Upload SARIF report - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: ${{ steps.scout.outputs.result-file }} - - release: - runs-on: ubuntu-24.04 - if: startsWith(github.ref, 'refs/tags/dockerfile') - needs: - - prepare - - test - - image - steps: - - - name: GitHub Release - uses: softprops/action-gh-release@c95fe1489396fe8a9eb87c0abf8aa5b2ef267fda # v2.2.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - draft: true - name: ${{ needs.prepare.outputs.tag }} diff --git a/.github/workflows/test-os.yml b/.github/workflows/test-os.yml deleted file mode 100644 index 692931aff7b7..000000000000 --- a/.github/workflows/test-os.yml +++ /dev/null @@ -1,261 +0,0 @@ -name: test-os - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - schedule: - - cron: '0 10 * * *' - workflow_dispatch: - push: - branches: - - 'master' - - 'v[0-9]+.[0-9]+' - pull_request: - paths-ignore: - - 'README.md' - - 'docs/**' - - 'frontend/dockerfile/docs/**' - -env: - GO_VERSION: "1.23" - SETUP_BUILDX_VERSION: "edge" - SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" - DESTDIR: "./bin" - -jobs: - build: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - platform: - - windows/amd64 - - freebsd/amd64 - steps: - - - name: Prepare - run: | - platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: ${{ env.SETUP_BUILDX_VERSION }} - driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} - buildkitd-flags: --debug - - - name: Build - uses: docker/bake-action@v6 - with: - provenance: false - targets: binaries-for-test - set: | - *.platform=${{ matrix.platform }} - *.cache-from=type=gha,scope=binaries-for-test-${{ env.PLATFORM_PAIR }} - *.cache-to=type=gha,scope=binaries-for-test-${{ env.PLATFORM_PAIR }},repository=${{ github.repository }},ghtoken=${{ secrets.GITHUB_TOKEN }} - - - name: List artifacts - run: | - tree -nh ${{ env.DESTDIR }} - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: buildkit-${{ env.PLATFORM_PAIR }} - path: ${{ env.DESTDIR }}/* - if-no-files-found: error - retention-days: 1 - - test-windows-amd64: - runs-on: ${{ matrix.os }} - needs: - - build - env: - TESTFLAGS: "-v --timeout=60m" - GOTESTSUM_FORMAT: "standard-verbose" - strategy: - fail-fast: false - matrix: - os: [windows-2022, windows-2025] - worker: - - containerd - pkg: - - ./client - - ./cmd/buildctl - - ./worker/containerd - - ./solver - - ./frontend - - ./frontend/dockerfile - include: - - os: windows-2022 - worker: containerd - pkg: ./... - skip-integration-tests: 1 - - os: windows-2025 - worker: containerd - pkg: ./... - skip-integration-tests: 1 - steps: - - - name: Prepare - run: | - echo "TEST_REPORT_NAME=${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV - testFlags="${{ env.TESTFLAGS }}" - if [ -n "${{ matrix.worker }}" ]; then - testFlags="${testFlags} --run=TestIntegration/.*/worker=${{ matrix.worker }}" - fi - echo "TESTFLAGS=${testFlags}" >> $GITHUB_ENV - shell: bash - - - name: Checkout - uses: actions/checkout@v4 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v3 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: "${{ env.GO_VERSION }}" - cache: false - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - name: buildkit-windows-amd64 - path: ${{ github.workspace }}\bin - - - name: Add bin folder to Path - run: | - echo "${{ github.workspace }}\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Test - env: - TMPDIR: ${{ runner.temp }} - SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }} - run: | - mkdir -p ./bin/testreports - gotestsum \ - --jsonfile="./bin/testreports/go-test-report-${{ env.TEST_REPORT_NAME }}.json" \ - --junitfile="./bin/testreports/junit-report-${{ env.TEST_REPORT_NAME }}.xml" \ - --packages="${{ matrix.pkg }}" \ - -- \ - "-mod=vendor" \ - "-coverprofile" "./bin/testreports/coverage-${{ env.TEST_REPORT_NAME }}.txt" \ - "-covermode" "atomic" ${{ env.TESTFLAGS }} - shell: bash - - - name: Send to Codecov - if: always() - uses: codecov/codecov-action@v5 - with: - directory: ./bin/testreports - env_vars: RUNNER_OS - flags: unit - disable_file_fixes: true - token: ${{ secrets.CODECOV_TOKEN }} # used to upload coverage reports: https://github.com/moby/buildkit/pull/4660#issue-2142122533 - - - name: Generate annotations - if: always() - uses: crazy-max/.github/.github/actions/gotest-annotations@fa6141aedf23596fb8bdcceab9cce8dadaa31bd9 - with: - directory: ./bin/testreports - - - name: Upload test reports - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-reports-${{ matrix.os }}-${{ env.TEST_REPORT_NAME }} - path: ./bin/testreports - retention-days: 1 - - - name: Dump context - if: failure() - uses: crazy-max/ghaction-dump-context@v2 - - test-freebsd-amd64: - runs-on: ubuntu-22.04 - needs: - - build - env: - GOOS: freebsd - steps: - - - name: Checkout - uses: actions/checkout@v4 - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - name: buildkit-freebsd-amd64 - path: ${{ env.DESTDIR }} - - - name: Cache Vagrant boxes - uses: actions/cache@v4 - with: - path: ~/.vagrant.d/boxes - key: ${{ runner.os }}-vagrant-${{ hashFiles('hack/Vagrantfile.freebsd') }} - restore-keys: | - ${{ runner.os }}-vagrant- - - - name: Install vagrant - run: | - set -x - wget -O - https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list - sudo apt-get update - sudo apt-get install -y libvirt-dev libvirt-daemon libvirt-daemon-system vagrant vagrant-libvirt ruby-libvirt - sudo systemctl enable --now libvirtd - sudo chmod a+rw /var/run/libvirt/libvirt-sock - vagrant plugin install vagrant-libvirt - vagrant --version - - - name: Set up vagrant - run: | - ln -sf hack/Vagrantfile.freebsd Vagrantfile - vagrant up --no-tty - - - name: Smoke test - uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 - with: - timeout_minutes: 20 - max_attempts: 5 - command: | - vagrant up --provision-with=test-smoke - - - name: BuildKit logs - if: always() - run: | - vagrant ssh -- "sudo cat /vagrant/.tmp/logs/buildkitd" - - - name: Containerd logs - if: always() - run: | - vagrant ssh -- "sudo cat /vagrant/.tmp/logs/containerd" - - sandbox-build: - runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} - strategy: - fail-fast: false - matrix: - platform: - - linux/amd64 - - linux/arm64 - steps: - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: ${{ env.SETUP_BUILDX_VERSION }} - driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} - buildkitd-flags: --debug - - - name: Build - uses: docker/bake-action@v6 - with: - targets: integration-tests-base - set: | - *.platform=${{ matrix.platform }} diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml deleted file mode 100644 index 4bc88855e236..000000000000 --- a/.github/workflows/validate.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: validate - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - push: - branches: - - 'master' - - 'v[0-9]+.[0-9]+' - tags: - - 'v*' - - 'dockerfile/*' - pull_request: - -env: - SETUP_BUILDX_VERSION: "edge" - SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" - -jobs: - prepare: - runs-on: ubuntu-24.04 - outputs: - targets: ${{ steps.generate.outputs.targets }} - steps: - - - name: Checkout - uses: actions/checkout@v4 - - - name: List targets - id: generate - uses: docker/bake-action/subaction/list-targets@v6 - with: - target: validate - - validate: - runs-on: ubuntu-24.04 - needs: - - prepare - strategy: - fail-fast: false - matrix: - target: ${{ fromJson(needs.prepare.outputs.targets) }} - steps: - - - name: Prepare - run: | - if [ "$GITHUB_REPOSITORY" = "moby/buildkit" ]; then - echo "GOLANGCI_LINT_MULTIPLATFORM=1" >> $GITHUB_ENV - fi - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: ${{ env.SETUP_BUILDX_VERSION }} - driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} - buildkitd-flags: --debug - - - name: Validate - uses: docker/bake-action@v6 - with: - targets: ${{ matrix.target }} - - archutil-arm64: - runs-on: ubuntu-24.04-arm - steps: - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: ${{ env.SETUP_BUILDX_VERSION }} - driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} - buildkitd-flags: --debug - - - name: Validate - uses: docker/bake-action@v6 - with: - targets: validate-archutil - set: | - *.platform=linux/arm64 From 5f670b0776fc7fc12c0f0b518a730f280c5d7d09 Mon Sep 17 00:00:00 2001 From: Pudong Zheng Date: Fri, 22 Sep 2023 08:06:25 +0000 Subject: [PATCH 19/23] update gitpod yaml --- .gitpod.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitpod.yml diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 000000000000..2429c1a1ca78 --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1 @@ +image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:at-node-18-17-1-gha.15610 \ No newline at end of file From 0fbaadb08076a742ef1e26e74ba66d7458f1eb19 Mon Sep 17 00:00:00 2001 From: Pudong Zheng Date: Fri, 22 Sep 2023 08:11:49 +0000 Subject: [PATCH 20/23] do not mirror on localhost --- util/resolver/resolver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/util/resolver/resolver.go b/util/resolver/resolver.go index 6dca73461116..6e8eb7abbe8a 100644 --- a/util/resolver/resolver.go +++ b/util/resolver/resolver.go @@ -125,6 +125,9 @@ func NewRegistryConfig(m map[string]config.RegistryConfig) docker.RegistryHosts mirrorHost := "localhost:8080" return docker.Registries( func(host string) ([]docker.RegistryHost, error) { + if ok, _ := docker.MatchLocalhost(host); ok { + return nil, nil + } c, ok := m[host] if !ok { c = config.RegistryConfig{} From ddc4da24e9b30971cc3f06f4b39872a90b38c5da Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Fri, 21 Mar 2025 02:07:11 +0000 Subject: [PATCH 21/23] Use a recent image Tool: gitpod/catfood.gitpod.cloud --- .gitpod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitpod.yml b/.gitpod.yml index 2429c1a1ca78..4c271f95e1b5 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1 +1 @@ -image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:at-node-18-17-1-gha.15610 \ No newline at end of file +image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:main-gha.30393 \ No newline at end of file From 9b2bf83317a23506b62a661349907a96d2e8cf7a Mon Sep 17 00:00:00 2001 From: Pudong Zheng Date: Fri, 21 Mar 2025 09:37:26 +0000 Subject: [PATCH 22/23] build on this repo Tool: gitpod/catfood.gitpod.cloud --- .github/workflows/buildkit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index c8e25c4d6d50..b2957ca9fbdd 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -55,7 +55,7 @@ jobs: PUSH=push fi fi - if [ "$GITHUB_REPOSITORY" != "moby/buildkit" ]; then + if [ "$GITHUB_REPOSITORY" != "gitpod-io/buildkit" ]; then PUSH=false fi echo "tag=${TAG}" >>${GITHUB_OUTPUT} From 93509e16a383c7e9baf619c704ae632b864196f1 Mon Sep 17 00:00:00 2001 From: Pudong Zheng Date: Tue, 8 Jul 2025 13:29:36 +0000 Subject: [PATCH 23/23] upgrade alpine --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 50a21cdb2d57..d5cd28b843e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ ARG GOTESTSUM_VERSION=v1.9.0 ARG DELVE_VERSION=v1.23.1 ARG GO_VERSION=1.23 -ARG ALPINE_VERSION=3.21 +ARG ALPINE_VERSION=3.22 ARG XX_VERSION=1.6.1 ARG BUILDKIT_DEBUG ARG EXPORT_BASE=alpine