diff --git a/CHANGELOG.md b/CHANGELOG.md index 113d70428..081d5ccce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Changed +- `umoci`'s `oci/cas` and `oci/config` libraries have been massively refactored + and rewritten, to allow for third-parties to use the OCI libraries. The plan + is for these to eventually become part of an OCI project. openSUSE/umoci#90 ## [0.1.0] - 2017-02-11 ### Added diff --git a/cmd/umoci/config.go b/cmd/umoci/config.go index d2c7834ca..d07d4f411 100644 --- a/cmd/umoci/config.go +++ b/cmd/umoci/config.go @@ -24,7 +24,7 @@ import ( "github.com/apex/log" "github.com/openSUSE/umoci/mutate" "github.com/openSUSE/umoci/oci/cas" - igen "github.com/openSUSE/umoci/oci/generate" + igen "github.com/openSUSE/umoci/oci/config/generate" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/urfave/cli" diff --git a/cmd/umoci/gc.go b/cmd/umoci/gc.go index 6cc0203f1..99fa8a475 100644 --- a/cmd/umoci/gc.go +++ b/cmd/umoci/gc.go @@ -19,6 +19,7 @@ package main import ( "github.com/openSUSE/umoci/oci/cas" + "github.com/openSUSE/umoci/oci/casext" "github.com/pkg/errors" "github.com/urfave/cli" "golang.org/x/net/context" @@ -56,8 +57,9 @@ func gc(ctx *cli.Context) error { if err != nil { return errors.Wrap(err, "open CAS") } + engineExt := casext.Engine{engine} defer engine.Close() // Run the GC. - return errors.Wrap(cas.GC(context.Background(), engine), "gc") + return errors.Wrap(engineExt.GC(context.Background()), "gc") } diff --git a/cmd/umoci/init.go b/cmd/umoci/init.go index 272a13b5c..393f3d26a 100644 --- a/cmd/umoci/init.go +++ b/cmd/umoci/init.go @@ -54,7 +54,7 @@ func initLayout(ctx *cli.Context) error { return errors.Wrap(err, "image layout creation") } - if err := cas.CreateLayout(imagePath); err != nil { + if err := cas.Create(imagePath); err != nil { return errors.Wrap(err, "image layout creation") } diff --git a/cmd/umoci/main.go b/cmd/umoci/main.go index 36f03a089..a6c089179 100644 --- a/cmd/umoci/main.go +++ b/cmd/umoci/main.go @@ -25,6 +25,9 @@ import ( logcli "github.com/apex/log/handlers/cli" "github.com/pkg/errors" "github.com/urfave/cli" + + // Include all official OCI images. + _ "github.com/openSUSE/umoci/oci/cas/drivers" ) // version is version ID for the source, read from VERSION in the source and diff --git a/cmd/umoci/new.go b/cmd/umoci/new.go index 675e974e7..613d13b0c 100644 --- a/cmd/umoci/new.go +++ b/cmd/umoci/new.go @@ -23,7 +23,7 @@ import ( "github.com/apex/log" "github.com/openSUSE/umoci/oci/cas" - igen "github.com/openSUSE/umoci/oci/generate" + igen "github.com/openSUSE/umoci/oci/config/generate" imeta "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" diff --git a/cmd/umoci/repack.go b/cmd/umoci/repack.go index 2714c7e16..58e873f34 100644 --- a/cmd/umoci/repack.go +++ b/cmd/umoci/repack.go @@ -28,7 +28,7 @@ import ( "github.com/openSUSE/umoci" "github.com/openSUSE/umoci/mutate" "github.com/openSUSE/umoci/oci/cas" - igen "github.com/openSUSE/umoci/oci/generate" + igen "github.com/openSUSE/umoci/oci/config/generate" "github.com/openSUSE/umoci/oci/layer" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" diff --git a/cmd/umoci/stat.go b/cmd/umoci/stat.go index d380fdb14..18d6698c1 100644 --- a/cmd/umoci/stat.go +++ b/cmd/umoci/stat.go @@ -23,6 +23,7 @@ import ( "os" "github.com/openSUSE/umoci/oci/cas" + "github.com/openSUSE/umoci/oci/casext" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/urfave/cli" @@ -63,6 +64,7 @@ func stat(ctx *cli.Context) error { if err != nil { return errors.Wrap(err, "open CAS") } + engineExt := casext.Engine{engine} defer engine.Close() manifestDescriptor, err := engine.GetReference(context.Background(), tagName) @@ -76,7 +78,7 @@ func stat(ctx *cli.Context) error { } // Get stat information. - ms, err := Stat(context.Background(), engine, manifestDescriptor) + ms, err := Stat(context.Background(), engineExt, manifestDescriptor) if err != nil { return errors.Wrap(err, "stat") } diff --git a/cmd/umoci/unpack.go b/cmd/umoci/unpack.go index 882f46dff..640ff2220 100644 --- a/cmd/umoci/unpack.go +++ b/cmd/umoci/unpack.go @@ -26,6 +26,7 @@ import ( "github.com/apex/log" "github.com/openSUSE/umoci" "github.com/openSUSE/umoci/oci/cas" + "github.com/openSUSE/umoci/oci/casext" "github.com/openSUSE/umoci/oci/layer" "github.com/openSUSE/umoci/pkg/idtools" ispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -125,15 +126,16 @@ func unpack(ctx *cli.Context) error { if err != nil { return errors.Wrap(err, "open CAS") } + engineExt := casext.Engine{engine} defer engine.Close() - fromDescriptor, err := engine.GetReference(context.Background(), fromName) + fromDescriptor, err := engineExt.GetReference(context.Background(), fromName) if err != nil { return errors.Wrap(err, "get descriptor") } meta.From = fromDescriptor - manifestBlob, err := cas.FromDescriptor(context.Background(), engine, meta.From) + manifestBlob, err := engineExt.FromDescriptor(context.Background(), meta.From) if err != nil { return errors.Wrap(err, "get manifest") } @@ -172,7 +174,7 @@ func unpack(ctx *cli.Context) error { // should be fixed once the CAS engine PR is merged into // image-tools. https://github.com/opencontainers/image-tools/pull/5 log.Info("unpacking bundle ...") - if err := layer.UnpackManifest(context.Background(), engine, bundlePath, manifest, &meta.MapOptions); err != nil { + if err := layer.UnpackManifest(context.Background(), engineExt, bundlePath, manifest, &meta.MapOptions); err != nil { return errors.Wrap(err, "create runtime bundle") } log.Info("... done") diff --git a/cmd/umoci/utils.go b/cmd/umoci/utils.go index e0318b99e..4f22c2f82 100644 --- a/cmd/umoci/utils.go +++ b/cmd/umoci/utils.go @@ -28,8 +28,8 @@ import ( "text/tabwriter" "github.com/docker/go-units" - "github.com/openSUSE/umoci/oci/cas" - igen "github.com/openSUSE/umoci/oci/generate" + "github.com/openSUSE/umoci/oci/casext" + igen "github.com/openSUSE/umoci/oci/config/generate" "github.com/openSUSE/umoci/oci/layer" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -180,7 +180,7 @@ type historyStat struct { // Stat computes the ManifestStat for a given manifest blob. The provided // descriptor must refer to an OCI Manifest. -func Stat(ctx context.Context, engine cas.Engine, manifestDescriptor ispec.Descriptor) (ManifestStat, error) { +func Stat(ctx context.Context, engine casext.Engine, manifestDescriptor ispec.Descriptor) (ManifestStat, error) { var stat ManifestStat if manifestDescriptor.MediaType != ispec.MediaTypeImageManifest { @@ -188,7 +188,7 @@ func Stat(ctx context.Context, engine cas.Engine, manifestDescriptor ispec.Descr } // We have to get the actual manifest. - manifestBlob, err := cas.FromDescriptor(ctx, engine, manifestDescriptor) + manifestBlob, err := engine.FromDescriptor(ctx, manifestDescriptor) if err != nil { return stat, err } @@ -199,7 +199,7 @@ func Stat(ctx context.Context, engine cas.Engine, manifestDescriptor ispec.Descr } // Now get the config. - configBlob, err := cas.FromDescriptor(ctx, engine, manifest.Config) + configBlob, err := engine.FromDescriptor(ctx, manifest.Config) if err != nil { return stat, errors.Wrap(err, "stat") } diff --git a/mutate/mutate.go b/mutate/mutate.go index 4efbd070a..3b2149efe 100644 --- a/mutate/mutate.go +++ b/mutate/mutate.go @@ -23,6 +23,7 @@ import ( "time" "github.com/openSUSE/umoci/oci/cas" + "github.com/openSUSE/umoci/oci/casext" "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -40,7 +41,7 @@ func manifestPtr(m ispec.Manifest) *ispec.Manifest { return &m } // TODO: Implement manifest list support. type Mutator struct { // These are the arguments we got in New(). - engine cas.Engine + engine casext.Engine source ispec.Descriptor // Cached values of the configuration and manifest. @@ -75,7 +76,7 @@ type Meta struct { func (m *Mutator) cache(ctx context.Context) error { // We need the manifest if m.manifest == nil { - blob, err := cas.FromDescriptor(ctx, m.engine, m.source) + blob, err := m.engine.FromDescriptor(ctx, m.source) if err != nil { return errors.Wrap(err, "cache source manifest") } @@ -92,7 +93,7 @@ func (m *Mutator) cache(ctx context.Context) error { } if m.config == nil { - blob, err := cas.FromDescriptor(ctx, m.engine, m.manifest.Config) + blob, err := m.engine.FromDescriptor(ctx, m.manifest.Config) if err != nil { return errors.Wrap(err, "cache source config") } @@ -120,7 +121,7 @@ func New(engine cas.Engine, src ispec.Descriptor) (*Mutator, error) { } return &Mutator{ - engine: engine, + engine: casext.Engine{engine}, source: src, }, nil } diff --git a/mutate/mutate_test.go b/mutate/mutate_test.go index e7bcda151..a1ad507cc 100644 --- a/mutate/mutate_test.go +++ b/mutate/mutate_test.go @@ -30,6 +30,9 @@ import ( imeta "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/net/context" + + // Include all known drivers. + _ "github.com/openSUSE/umoci/oci/cas/drivers" ) // These come from just running the code. @@ -41,7 +44,7 @@ const ( func setup(t *testing.T, dir string) (cas.Engine, ispec.Descriptor) { dir = filepath.Join(dir, "image") - if err := cas.CreateLayout(dir); err != nil { + if err := cas.Create(dir); err != nil { t.Fatal(err) } diff --git a/oci/cas/cas.go b/oci/cas/cas.go index b75bb3f31..db679e674 100644 --- a/oci/cas/cas.go +++ b/oci/cas/cas.go @@ -20,8 +20,6 @@ package cas import ( "fmt" "io" - "os" - "path/filepath" // We need to include sha256 in order for go-digest to properly handle such // hashes, since Go's crypto library like to lazy-load cryptographic @@ -30,7 +28,6 @@ import ( "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" "golang.org/x/net/context" ) @@ -38,16 +35,6 @@ const ( // BlobAlgorithm is the name of the only supported digest algorithm for blobs. // FIXME: We can make this a list. BlobAlgorithm = digest.SHA256 - - // refDirectory is the directory inside an OCI image that contains references. - refDirectory = "refs" - - // blobDirectory is the directory inside an OCI image that contains blobs. - blobDirectory = "blobs" - - // layoutFile is the file in side an OCI image the indicates what version - // of the OCI spec the image is. - layoutFile = "oci-layout" ) // Exposed errors. @@ -77,6 +64,11 @@ type Engine interface { // as the reader. Note that due to intricacies in the Go JSON // implementation, we cannot guarantee that two calls to PutBlobJSON() will // return the same digest. + // + // TODO: Use a proper JSON serialisation library, which actually guarantees + // consistent output. Go's JSON library doesn't even attempt to sort + // map[...]... objects (which have their iteration order randomised + // in Go). PutBlobJSON(ctx context.Context, data interface{}) (digest digest.Digest, size int64, err error) // PutReference adds a new reference descriptor blob to the image. This is @@ -110,52 +102,13 @@ type Engine interface { // ListReferences returns the set of reference names stored in the image. ListReferences(ctx context.Context) (names []string, err error) - // GC executes a garbage collection of any non-blob garbage in the store + // Clean executes a garbage collection of any non-blob garbage in the store // (this includes temporary files and directories not reachable from the // CAS interface). This MUST NOT remove any blobs or references in the // store. - GC(ctx context.Context) (err error) + Clean(ctx context.Context) (err error) // Close releases all references held by the engine. Subsequent operations // may fail. Close() (err error) } - -// Open will create an Engine reference to the OCI image at the provided -// path. If the image format is not supported, ErrNotImplemented will be -// returned. If the path does not exist, os.ErrNotExist will be returned. -func Open(path string) (Engine, error) { - fi, err := os.Stat(path) - if err != nil { - return nil, err - } - - if fi.IsDir() { - return newDirEngine(path) - } - - return nil, ErrNotImplemented -} - -// blobPath returns the path to a blob given its digest, relative to the root -// of the OCI image. The digest must be of the form algorithm:hex. -func blobPath(digest digest.Digest) (string, error) { - if err := digest.Validate(); err != nil { - return "", errors.Wrapf(err, "invalid digest: %q", digest) - } - - algo := digest.Algorithm() - hash := digest.Hex() - - if algo != BlobAlgorithm { - return "", errors.Errorf("unsupported algorithm: %q", algo) - } - - return filepath.Join(blobDirectory, algo.String(), hash), nil -} - -// refPath returns the path to a reference given its name, relative to the r -// oot of the OCI image. -func refPath(name string) (string, error) { - return filepath.Join(refDirectory, name), nil -} diff --git a/oci/cas/drivers.go b/oci/cas/drivers.go new file mode 100644 index 000000000..8968b134d --- /dev/null +++ b/oci/cas/drivers.go @@ -0,0 +1,101 @@ +/* + * umoci: Umoci Modifies Open Containers' Images + * Copyright (C) 2017 SUSE LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cas + +import ( + "sync" + + "github.com/pkg/errors" +) + +// TODO: URIs need to be handled better, with some way of specifying what the +// format or protocol is meant to be used. Currently Create(...) doesn't +// work properly. + +// Driver is an interface describing a CAS driver that can be used to create +// new cas.Engine instances. The intention is for this to be generic enough +// that multiple backends can be implemented for umoci and other tools without +// requiring changes to other components of such tools. +type Driver interface { + // Supported returns whether the resource at the given URI is supported by + // the driver (used for auto-detection). If two drivers support the same + // URI, then the earliest registered driver takes precedence. + // + // Note that this is _not_ a validation of the URI -- if the URI refers to + // an invalid or non-existent resource it is expected that the URI is + // "supported". + Supported(uri string) bool + + // Open "opens" a new CAS engine accessor for the given URI. + Open(uri string) (Engine, error) + + // Create creates a new image at the provided URI. + Create(uri string) error +} + +var ( + dm sync.RWMutex + drivers []Driver +) + +// Register registers a new Driver in the global set of drivers. This is +// intended to be called from the init function in packages that implement +// cas.Engine (similar to the crypto package). +func Register(driver Driver) { + dm.Lock() + drivers = append(drivers, driver) + dm.Unlock() +} + +func findSupported(uri string) Driver { + dm.RLock() + defer dm.RUnlock() + + for _, driver := range drivers { + if driver.Supported(uri) { + return driver + } + } + return nil +} + +// Open returns a new cas.Engine created by one of the registered drivers that +// support the provided URI (if no such driver exists, an error is returned). +// If more than one driver supports the provided URI, the first of the +// candidate drivers to have been registered is chosen. +func Open(uri string) (Engine, error) { + driver := findSupported(uri) + if driver == nil { + return nil, errors.Errorf("drivers: unsupported uri: %s", uri) + } + + return driver.Open(uri) +} + +// Create creates a new image by one of the registered drivers that support the +// provided URI (if no such driver exists, an error is returned). If more than +// one driver supports the provided URI, the first of the candidate drivers to +// be registered is chosen. +func Create(uri string) error { + driver := findSupported(uri) + if driver == nil { + return errors.Errorf("drivers: unsupported uri: %s", uri) + } + + return driver.Create(uri) +} diff --git a/oci/cas/drivers/all.go b/oci/cas/drivers/all.go new file mode 100644 index 000000000..e3634b97b --- /dev/null +++ b/oci/cas/drivers/all.go @@ -0,0 +1,27 @@ +/* + * umoci: Umoci Modifies Open Containers' Images + * Copyright (C) 2017 SUSE LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package drivers is an empty package which has subpackages that implement +// cas.Drivers (and register said drivers with cas). Importing this package +// will register all official OCI cas drivers. +package drivers + +// Import all official OCI drivers. +import ( + // Implements directory-backed OCI layouts. + _ "github.com/openSUSE/umoci/oci/cas/drivers/dir" +) diff --git a/oci/cas/cas_test.go b/oci/cas/drivers/dir/cas_test.go similarity index 96% rename from oci/cas/cas_test.go rename to oci/cas/drivers/dir/cas_test.go index 821526d3f..80cd966f7 100644 --- a/oci/cas/cas_test.go +++ b/oci/cas/drivers/dir/cas_test.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package cas +package dir import ( "bytes" @@ -27,6 +27,7 @@ import ( "reflect" "testing" + "github.com/openSUSE/umoci/oci/cas" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "golang.org/x/net/context" @@ -45,7 +46,7 @@ func TestCreateLayout(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := CreateLayout(image); err != nil { + if err := Create(image); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -68,7 +69,7 @@ func TestCreateLayout(t *testing.T) { } // We should get an error if we try to create a new image atop an old one. - if err := CreateLayout(image); err == nil { + if err := Create(image); err == nil { t.Errorf("expected to get a cowardly no-clobber error!") } } @@ -83,7 +84,7 @@ func TestEngineBlob(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := CreateLayout(image); err != nil { + if err := Create(image); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -100,7 +101,7 @@ func TestEngineBlob(t *testing.T) { {[]byte("some blob")}, {[]byte("another blob")}, } { - digester := BlobAlgorithm.Digester() + digester := cas.BlobAlgorithm.Digester() if _, err := io.Copy(digester.Hash(), bytes.NewReader(test.bytes)); err != nil { t.Fatalf("could not hash bytes: %+v", err) } @@ -169,7 +170,7 @@ func TestEngineBlobJSON(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := CreateLayout(image); err != nil { + if err := Create(image); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -252,7 +253,7 @@ func TestEngineReference(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := CreateLayout(image); err != nil { + if err := Create(image); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -309,7 +310,7 @@ func TestEngineValidate(t *testing.T) { } defer os.RemoveAll(root) - var engine Engine + var engine cas.Engine var image string // Empty directory. @@ -359,7 +360,7 @@ func TestEngineValidate(t *testing.T) { if err := os.Remove(image); err != nil { t.Fatal(err) } - if err := CreateLayout(image); err != nil { + if err := Create(image); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } if err := os.RemoveAll(filepath.Join(image, blobDirectory)); err != nil { @@ -379,7 +380,7 @@ func TestEngineValidate(t *testing.T) { if err := os.Remove(image); err != nil { t.Fatal(err) } - if err := CreateLayout(image); err != nil { + if err := Create(image); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } if err := os.RemoveAll(filepath.Join(image, blobDirectory)); err != nil { @@ -402,7 +403,7 @@ func TestEngineValidate(t *testing.T) { if err := os.Remove(image); err != nil { t.Fatal(err) } - if err := CreateLayout(image); err != nil { + if err := Create(image); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } if err := os.RemoveAll(filepath.Join(image, refDirectory)); err != nil { @@ -422,7 +423,7 @@ func TestEngineValidate(t *testing.T) { if err := os.Remove(image); err != nil { t.Fatal(err) } - if err := CreateLayout(image); err != nil { + if err := Create(image); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } if err := os.RemoveAll(filepath.Join(image, refDirectory)); err != nil { diff --git a/oci/cas/dir.go b/oci/cas/drivers/dir/dir.go similarity index 82% rename from oci/cas/dir.go rename to oci/cas/drivers/dir/dir.go index 155868e7d..2e09aab76 100644 --- a/oci/cas/dir.go +++ b/oci/cas/drivers/dir/dir.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package cas +package dir import ( "bytes" @@ -26,6 +26,7 @@ import ( "path/filepath" "reflect" + "github.com/openSUSE/umoci/oci/cas" "github.com/openSUSE/umoci/pkg/system" "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -33,55 +34,45 @@ import ( "golang.org/x/net/context" ) -// ImageLayoutVersion is the version of the image layout we support. This value -// is *not* the same as imagespec.Version, and the meaning of this field is -// still under discussion in the spec. For now we'll just hardcode the value -// and hope for the best. -const ImageLayoutVersion = "1.0.0" +const ( + // ImageLayoutVersion is the version of the image layout we support. This + // value is *not* the same as imagespec.Version, and the meaning of this + // field is still under discussion in the spec. For now we'll just hardcode + // the value and hope for the best. + ImageLayoutVersion = "1.0.0" -// CreateLayout creates a new OCI image layout at the given path. If the path -// already exists, os.ErrExist is returned. However, all of the parent -// components of the path will be created if necessary. -func CreateLayout(path string) error { - // We need to fail if path already exists, but we first create all of the - // parent paths. - dir := filepath.Dir(path) - if dir != "." { - if err := os.MkdirAll(dir, 0755); err != nil { - return errors.Wrap(err, "mkdir parent") - } - } - if err := os.Mkdir(path, 0755); err != nil { - return errors.Wrap(err, "mkdir") - } + // refDirectory is the directory inside an OCI image that contains references. + refDirectory = "refs" - // Create the necessary directories and "oci-layout" file. - if err := os.Mkdir(filepath.Join(path, blobDirectory), 0755); err != nil { - return errors.Wrap(err, "mkdir blobdir") - } - if err := os.Mkdir(filepath.Join(path, blobDirectory, BlobAlgorithm.String()), 0755); err != nil { - return errors.Wrap(err, "mkdir algorithm") - } - if err := os.Mkdir(filepath.Join(path, refDirectory), 0755); err != nil { - return errors.Wrap(err, "mkdir refdir") - } + // blobDirectory is the directory inside an OCI image that contains blobs. + blobDirectory = "blobs" - fh, err := os.Create(filepath.Join(path, layoutFile)) - if err != nil { - return errors.Wrap(err, "create oci-layout") - } - defer fh.Close() + // layoutFile is the file in side an OCI image the indicates what version + // of the OCI spec the image is. + layoutFile = "oci-layout" +) - ociLayout := &ispec.ImageLayout{ - Version: ImageLayoutVersion, +// blobPath returns the path to a blob given its digest, relative to the root +// of the OCI image. The digest must be of the form algorithm:hex. +func blobPath(digest digest.Digest) (string, error) { + if err := digest.Validate(); err != nil { + return "", errors.Wrapf(err, "invalid digest: %q", digest) } - if err := json.NewEncoder(fh).Encode(ociLayout); err != nil { - return errors.Wrap(err, "encode oci-layout") + algo := digest.Algorithm() + hash := digest.Hex() + + if algo != cas.BlobAlgorithm { + return "", errors.Errorf("unsupported algorithm: %q", algo) } - // Everything is now set up. - return nil + return filepath.Join(blobDirectory, algo.String(), hash), nil +} + +// refPath returns the path to a reference given its name, relative to the r +// oot of the OCI image. +func refPath(name string) (string, error) { + return filepath.Join(refDirectory, name), nil } type dirEngine struct { @@ -119,7 +110,7 @@ func (e *dirEngine) validate() error { content, err := ioutil.ReadFile(filepath.Join(e.path, layoutFile)) if err != nil { if os.IsNotExist(err) { - err = ErrInvalid + err = cas.ErrInvalid } return errors.Wrap(err, "read oci-layout") } @@ -132,29 +123,29 @@ func (e *dirEngine) validate() error { // XXX: Currently the meaning of this field is not adequately defined by // the spec, nor is the "official" value determined by the spec. if ociLayout.Version != ImageLayoutVersion { - return errors.Wrap(ErrInvalid, "layout version is supported") + return errors.Wrap(cas.ErrInvalid, "layout version is supported") } // Check that "blobs" and "refs" exist in the image. - // FIXME: We also should check that blobs *only* contains a BlobAlgorithm + // FIXME: We also should check that blobs *only* contains a cas.BlobAlgorithm // directory (with no subdirectories) and that refs *only* contains // files (optionally also making sure they're all JSON descriptors). if fi, err := os.Stat(filepath.Join(e.path, blobDirectory)); err != nil { if os.IsNotExist(err) { - err = ErrInvalid + err = cas.ErrInvalid } return errors.Wrap(err, "check blobdir") } else if !fi.IsDir() { - return errors.Wrap(ErrInvalid, "blobdir is directory") + return errors.Wrap(cas.ErrInvalid, "blobdir is directory") } if fi, err := os.Stat(filepath.Join(e.path, refDirectory)); err != nil { if os.IsNotExist(err) { - err = ErrInvalid + err = cas.ErrInvalid } return errors.Wrap(err, "check refdir") } else if !fi.IsDir() { - return errors.Wrap(ErrInvalid, "refdir is directory") + return errors.Wrap(cas.ErrInvalid, "refdir is directory") } return nil @@ -168,7 +159,7 @@ func (e *dirEngine) PutBlob(ctx context.Context, reader io.Reader) (digest.Diges return "", -1, errors.Wrap(err, "ensure tempdir") } - digester := BlobAlgorithm.Digester() + digester := cas.BlobAlgorithm.Digester() // We copy this into a temporary file because we need to get the blob hash, // but also to avoid half-writing an invalid blob. @@ -227,7 +218,7 @@ func (e *dirEngine) PutReference(ctx context.Context, name string, descriptor is if oldDescriptor, err := e.GetReference(ctx, name); err == nil { // We should not return an error if the two descriptors are identical. if !reflect.DeepEqual(oldDescriptor, descriptor) { - return ErrClobber + return cas.ErrClobber } return nil } else if !os.IsNotExist(errors.Cause(err)) { @@ -331,7 +322,7 @@ func (e *dirEngine) DeleteReference(ctx context.Context, name string) error { // ListBlobs returns the set of blob digests stored in the image. func (e *dirEngine) ListBlobs(ctx context.Context) ([]digest.Digest, error) { digests := []digest.Digest{} - blobDir := filepath.Join(e.path, blobDirectory, BlobAlgorithm.String()) + blobDir := filepath.Join(e.path, blobDirectory, cas.BlobAlgorithm.String()) if err := filepath.Walk(blobDir, func(path string, _ os.FileInfo, _ error) error { // Skip the actual directory. @@ -340,7 +331,7 @@ func (e *dirEngine) ListBlobs(ctx context.Context) ([]digest.Digest, error) { } // XXX: Do we need to handle multiple-directory-deep cases? - digest := digest.NewDigestFromHex(BlobAlgorithm.String(), filepath.Base(path)) + digest := digest.NewDigestFromHex(cas.BlobAlgorithm.String(), filepath.Base(path)) digests = append(digests, digest) return nil }); err != nil { @@ -371,10 +362,10 @@ func (e *dirEngine) ListReferences(ctx context.Context) ([]string, error) { return refs, nil } -// GC executes a garbage collection of any non-blob garbage in the store (this -// includes temporary files and directories not reachable from the CAS +// Clean executes a garbage collection of any non-blob garbage in the store +// (this includes temporary files and directories not reachable from the CAS // interface). This MUST NOT remove any blobs or references in the store. -func (e *dirEngine) GC(ctx context.Context) error { +func (e *dirEngine) Clean(ctx context.Context) error { // Effectively we are going to remove every directory except the standard // directories, unless they have a lock already. fh, err := os.Open(e.path) @@ -436,7 +427,9 @@ func (e *dirEngine) Close() error { return nil } -func newDirEngine(path string) (*dirEngine, error) { +// Open opens a new reference to the directory-backed OCI image referenced by +// the provided path. +func Open(path string) (cas.Engine, error) { engine := &dirEngine{ path: path, temp: "", @@ -448,3 +441,48 @@ func newDirEngine(path string) (*dirEngine, error) { return engine, nil } + +// Create creates a new OCI image layout at the given path. If the path already +// exists, os.ErrExist is returned. However, all of the parent components of +// the path will be created if necessary. +func Create(path string) error { + // We need to fail if path already exists, but we first create all of the + // parent paths. + dir := filepath.Dir(path) + if dir != "." { + if err := os.MkdirAll(dir, 0755); err != nil { + return errors.Wrap(err, "mkdir parent") + } + } + if err := os.Mkdir(path, 0755); err != nil { + return errors.Wrap(err, "mkdir") + } + + // Create the necessary directories and "oci-layout" file. + if err := os.Mkdir(filepath.Join(path, blobDirectory), 0755); err != nil { + return errors.Wrap(err, "mkdir blobdir") + } + if err := os.Mkdir(filepath.Join(path, blobDirectory, cas.BlobAlgorithm.String()), 0755); err != nil { + return errors.Wrap(err, "mkdir algorithm") + } + if err := os.Mkdir(filepath.Join(path, refDirectory), 0755); err != nil { + return errors.Wrap(err, "mkdir refdir") + } + + fh, err := os.Create(filepath.Join(path, layoutFile)) + if err != nil { + return errors.Wrap(err, "create oci-layout") + } + defer fh.Close() + + ociLayout := &ispec.ImageLayout{ + Version: ImageLayoutVersion, + } + + if err := json.NewEncoder(fh).Encode(ociLayout); err != nil { + return errors.Wrap(err, "encode oci-layout") + } + + // Everything is now set up. + return nil +} diff --git a/oci/cas/dir_test.go b/oci/cas/drivers/dir/dir_test.go similarity index 96% rename from oci/cas/dir_test.go rename to oci/cas/drivers/dir/dir_test.go index f3dccdcb4..4fd8b746c 100644 --- a/oci/cas/dir_test.go +++ b/oci/cas/drivers/dir/dir_test.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package cas +package dir import ( "bytes" @@ -28,6 +28,7 @@ import ( "syscall" "testing" + "github.com/openSUSE/umoci/oci/cas" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "golang.org/x/net/context" @@ -75,7 +76,7 @@ func TestCreateLayoutReadonly(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := CreateLayout(image); err != nil { + if err := Create(image); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -112,7 +113,7 @@ func TestEngineBlobReadonly(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := CreateLayout(image); err != nil { + if err := Create(image); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -128,7 +129,7 @@ func TestEngineBlobReadonly(t *testing.T) { t.Fatalf("unexpected error opening image: %+v", err) } - digester := BlobAlgorithm.Digester() + digester := cas.BlobAlgorithm.Digester() if _, err := io.Copy(digester.Hash(), bytes.NewReader(test.bytes)); err != nil { t.Fatalf("could not hash bytes: %+v", err) } @@ -198,7 +199,7 @@ func TestEngineBlobJSONReadonly(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := CreateLayout(image); err != nil { + if err := Create(image); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -281,7 +282,7 @@ func TestEngineReferenceReadonly(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := CreateLayout(image); err != nil { + if err := Create(image); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -351,7 +352,7 @@ func TestEngineGCLocking(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := CreateLayout(image); err != nil { + if err := Create(image); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -363,7 +364,7 @@ func TestEngineGCLocking(t *testing.T) { t.Fatalf("unexpected error opening image: %+v", err) } - digester := BlobAlgorithm.Digester() + digester := cas.BlobAlgorithm.Digester() if _, err := io.Copy(digester.Hash(), bytes.NewReader(content)); err != nil { t.Fatalf("could not hash bytes: %+v", err) } @@ -397,7 +398,8 @@ func TestEngineGCLocking(t *testing.T) { t.Fatalf("unexpected error opening image: %+v", err) } - if err := GC(ctx, gcEngine); err != nil { + // TODO: This should be done with casext.GC... + if err := gcEngine.Clean(ctx); err != nil { t.Fatalf("unexpected error while GCing image: %+v", err) } diff --git a/oci/cas/drivers/dir/driver.go b/oci/cas/drivers/dir/driver.go new file mode 100644 index 000000000..db7945e1b --- /dev/null +++ b/oci/cas/drivers/dir/driver.go @@ -0,0 +1,62 @@ +/* + * umoci: Umoci Modifies Open Containers' Images + * Copyright (C) 2017 SUSE LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dir + +import ( + "os" + + "github.com/openSUSE/umoci/oci/cas" +) + +// Driver is an implementation of drivers.Driver for local directory-backed OCI +// image layouts. +var Driver cas.Driver = dirDriver{} + +type dirDriver struct{} + +// Supported returns whether the resource at the given URI is supported by the +// driver (used for auto-detection). If two drivers support the same URI, then +// the earliest registered driver takes precedence. +// +// Note that this is _not_ a validation of the URI -- if the URI refers to an +// invalid or non-existent resource it is expected that the URI is "supported". +func (d dirDriver) Supported(uri string) bool { + fi, err := os.Stat(uri) + if err != nil { + // If we got an error, we only support it if the error is that the + // target doesn't exist -- Create handles creating the necessary + // directories. + return os.IsNotExist(err) + } + // dir stands for directory + return fi.IsDir() +} + +// Open "opens" a new CAS engine accessor for the given URI. +func (d dirDriver) Open(uri string) (cas.Engine, error) { + return Open(uri) +} + +// Create creates a new image at the provided URI. +func (d dirDriver) Create(uri string) error { + return Create(uri) +} + +func init() { + cas.Register(Driver) +} diff --git a/oci/cas/blob.go b/oci/casext/blob.go similarity index 94% rename from oci/cas/blob.go rename to oci/casext/blob.go index 6f34951cf..124b8f2a0 100644 --- a/oci/cas/blob.go +++ b/oci/casext/blob.go @@ -15,13 +15,14 @@ * limitations under the License. */ -package cas +package casext import ( "encoding/json" "fmt" "io" + "github.com/openSUSE/umoci/oci/cas" "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -53,7 +54,7 @@ type Blob struct { Data interface{} } -func (b *Blob) load(ctx context.Context, engine Engine) error { +func (b *Blob) load(ctx context.Context, engine cas.Engine) error { reader, err := engine.GetBlob(ctx, b.Digest) if err != nil { return errors.Wrap(err, "get blob") @@ -134,14 +135,14 @@ func (b *Blob) Close() { } // FromDescriptor parses the blob referenced by the given descriptor. -func FromDescriptor(ctx context.Context, engine Engine, descriptor ispec.Descriptor) (*Blob, error) { +func (e Engine) FromDescriptor(ctx context.Context, descriptor ispec.Descriptor) (*Blob, error) { blob := &Blob{ MediaType: descriptor.MediaType, Digest: descriptor.Digest, Data: nil, } - if err := blob.load(ctx, engine); err != nil { + if err := blob.load(ctx, e); err != nil { return nil, errors.Wrap(err, "load") } diff --git a/oci/casext/casext.go b/oci/casext/casext.go new file mode 100644 index 000000000..4f752ba3f --- /dev/null +++ b/oci/casext/casext.go @@ -0,0 +1,29 @@ +/* + * umoci: Umoci Modifies Open Containers' Images + * Copyright (C) 2017 SUSE LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package casext provides extensions to the standard cas.Engine interface, +// allowing for generic functionality to be used on top of any implementation +// of cas.Engine. +package casext + +import "github.com/openSUSE/umoci/oci/cas" + +// Engine is a wrapper around cas.Engine that provides additional, generic +// extensions to the transport-dependent cas.Engine implementation. +type Engine struct { + cas.Engine +} diff --git a/oci/casext/gc.go b/oci/casext/gc.go new file mode 100644 index 000000000..347419dfc --- /dev/null +++ b/oci/casext/gc.go @@ -0,0 +1,102 @@ +/* + * umoci: Umoci Modifies Open Containers' Images + * Copyright (C) 2016, 2017 SUSE LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package casext + +import ( + "github.com/apex/log" + "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "golang.org/x/net/context" +) + +// GC will perform a mark-and-sweep garbage collection of the OCI image +// referenced by the given CAS engine. The root set is taken to be the set of +// references stored in the image, and all blobs not reachable by following a +// descriptor path from the root set will be removed. +// +// GC will only call ListBlobs and ListReferences once, and assumes that there +// is no change in the set of references or blobs after calling those +// functions. In other words, it assumes it is the only user of the image that +// is making modifications. Things will not go well if this assumption is +// challenged. +func (e Engine) GC(ctx context.Context) error { + // Generate the root set of descriptors. + var root []ispec.Descriptor + + names, err := e.ListReferences(ctx) + if err != nil { + return errors.Wrap(err, "get roots") + } + + for _, name := range names { + descriptor, err := e.GetReference(ctx, name) + if err != nil { + return errors.Wrapf(err, "get root %s", name) + } + log.WithFields(log.Fields{ + "name": name, + "digest": descriptor.Digest, + }).Debugf("GC: got reference") + root = append(root, descriptor) + } + + // Mark from the root sets. + black := map[digest.Digest]struct{}{} + for idx, descriptor := range root { + log.WithFields(log.Fields{ + "digest": descriptor.Digest, + }).Debugf("GC: marking from root") + + reachables, err := e.Reachable(ctx, descriptor) + if err != nil { + return errors.Wrapf(err, "getting reachables from root %d", idx) + } + for _, reachable := range reachables { + black[reachable] = struct{}{} + } + } + + // Sweep all blobs in the white set. + blobs, err := e.ListBlobs(ctx) + if err != nil { + return errors.Wrap(err, "get blob list") + } + + n := 0 + for _, digest := range blobs { + if _, ok := black[digest]; ok { + // Digest is in the black set. + continue + } + log.Infof("garbage collecting blob: %s", digest) + + if err := e.DeleteBlob(ctx, digest); err != nil { + return errors.Wrapf(err, "remove unmarked blob %s", digest) + } + n++ + } + + // Finally, tell CAS to GC it. + if err := e.Clean(ctx); err != nil { + return errors.Wrapf(err, "clean engine") + } + + log.Debugf("garbage collected %d blobs", n) + return nil +} diff --git a/oci/cas/gc.go b/oci/casext/walk.go similarity index 55% rename from oci/cas/gc.go rename to oci/casext/walk.go index 0b76b5994..d6c8e0531 100644 --- a/oci/cas/gc.go +++ b/oci/casext/walk.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package cas +package casext import ( "reflect" @@ -23,11 +23,10 @@ import ( "github.com/apex/log" "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" "golang.org/x/net/context" ) -// Used by gcState.mark() to determine which struct members are descriptors to +// Used by walkState.mark() to determine which struct members are descriptors to // recurse into them. We aren't interested in struct members which are not // either a slice of ispec.Descriptor or ispec.Descriptor themselves. var descriptorType = reflect.TypeOf(ispec.Descriptor{}) @@ -119,124 +118,105 @@ func childDescriptors(i interface{}) []ispec.Descriptor { // Unreachable. } -// gcState represents the state of the garbage collector at one point in time. -type gcState struct { +// walkState stores state information about the recursion into a given +// descriptor tree. +type walkState struct { // engine is the CAS engine we are operating on. engine Engine - // black is the set of digests which are reachable by a descriptor path - // from the root set. These are blobs which will *not* be deleted. The - // white set of digests is not stored in the state (we only have to compute - // it once anyway). - black map[digest.Digest]struct{} + // walkFunc is the WalkFunc provided by the user. + walkFunc WalkFunc } -func (gc *gcState) mark(ctx context.Context, descriptor ispec.Descriptor) error { +// TODO: Also provide Blob to WalkFunc so that callers don't need to load blobs +// more than once. This is quite important for remote CAS implementations. + +// TODO: Move this and blob.go to a separate package. + +// TODO: Implement an equivalent to filepath.SkipDir. + +// WalkFunc is the type of function passed to Walk. It will be a called on each +// descriptor encountered, recursively -- which may involve the function being +// called on the same descriptor multiple times (though because an OCI image is +// a Merkle tree there will never be any loops). If an error is returned by +// WalkFunc, the recursion will halt and the error will bubble up to the +// caller. +type WalkFunc func(descriptor ispec.Descriptor) error + +func (ws *walkState) recurse(ctx context.Context, descriptor ispec.Descriptor) error { log.WithFields(log.Fields{ "digest": descriptor.Digest, - }).Debugf("gc.mark") + }).Debugf("-> ws.recurse") - // Technically we should never hit this because you can't have cycles in a - // Merkle tree. But you can't be too careful. - if _, ok := gc.black[descriptor.Digest]; ok { - return nil + // Run walkFunc. + if err := ws.walkFunc(descriptor); err != nil { + return err } - // Add the descriptor itself to the black list. - gc.black[descriptor.Digest] = struct{}{} - - // Get the blob to recurse into. - blob, err := FromDescriptor(ctx, gc.engine, descriptor) + // Get blob to recurse into. + blob, err := ws.engine.FromDescriptor(ctx, descriptor) if err != nil { return err } + defer blob.Close() - // Mark all children. + // Recurse into children. for _, child := range childDescriptors(blob.Data) { - log.WithFields(log.Fields{ - "digest": descriptor.Digest, - "child": child.Digest, - }).Debugf("gc.mark recursing into child") - - if err := gc.mark(ctx, child); err != nil { + if err := ws.recurse(ctx, child); err != nil { return err } } + log.WithFields(log.Fields{ + "digest": descriptor.Digest, + }).Debugf("<- ws.recurse") return nil } -// GC will perform a mark-and-sweep garbage collection of the OCI image -// referenced by the given CAS engine. The root set is taken to be the set of -// references stored in the image, and all blobs not reachable by following a -// descriptor path from the root set will be removed. -// -// GC will only call ListBlobs and ListReferences once, and assumes that there -// is no change in the set of references or blobs after calling those -// functions. In other words, it assumes it is the only user of the image that -// is making modifications. Things will not go well if this assumption is -// challenged. -func GC(ctx context.Context, engine Engine) error { - // Generate the root set of descriptors. - var root []ispec.Descriptor - - names, err := engine.ListReferences(ctx) - if err != nil { - return errors.Wrap(err, "get roots") - } - - for _, name := range names { - descriptor, err := engine.GetReference(ctx, name) - if err != nil { - return errors.Wrapf(err, "get root %s", name) - } - log.WithFields(log.Fields{ - "name": name, - "digest": descriptor.Digest, - }).Debugf("GC: got reference") - root = append(root, descriptor) - } - - // Mark from the root set. - gc := &gcState{ - engine: engine, - black: map[digest.Digest]struct{}{}, +// Walk preforms a depth-first walk from a given root descriptor, using the +// provided CAS engine to fetch all other necessary descriptors. If an error is +// returned by the provided WalkFunc, walking is terminated and the error is +// returned to the caller. +func (e Engine) Walk(ctx context.Context, root ispec.Descriptor, walkFunc WalkFunc) error { + ws := &walkState{ + engine: e, + walkFunc: walkFunc, } + return ws.recurse(ctx, root) +} - for idx, descriptor := range root { - log.WithFields(log.Fields{ - "digest": descriptor.Digest, - }).Debugf("GC: marking from root") - if err := gc.mark(ctx, descriptor); err != nil { - return errors.Wrapf(err, "marking root %d", idx) - } - } +// Paths returns the set of descriptors that can be traversed from the provided +// root descriptor. It is effectively shorthand for Walk(). Note that there may +// be repeated descriptors in the returned slice, due to different blobs +// containing the same (or a similar) descriptor. +func (e Engine) Paths(ctx context.Context, root ispec.Descriptor) ([]ispec.Descriptor, error) { + var reachable []ispec.Descriptor - // Sweep all blobs in the white set. - blobs, err := engine.ListBlobs(ctx) - if err != nil { - return errors.Wrap(err, "get blob list") - } + err := e.Walk(ctx, root, func(descriptor ispec.Descriptor) error { + reachable = append(reachable, descriptor) + return nil + }) + return reachable, err +} - n := 0 - for _, digest := range blobs { - if _, ok := gc.black[digest]; ok { - // Digest is in the black set. - continue - } - log.Infof("garbage collecting blob: %s", digest) +// Reachable returns the set of digests which can be reached using a descriptor +// path from the provided root descriptor. It is effectively a shorthand for +// Walk(). The returned slice will *not* contain any duplicate digest.Digest +// entries. Note that without descriptors, a digest is not particularly +// meaninful (OCI blobs are not self-descriptive). +func (e Engine) Reachable(ctx context.Context, root ispec.Descriptor) ([]digest.Digest, error) { + seen := map[digest.Digest]struct{}{} - if err := engine.DeleteBlob(ctx, digest); err != nil { - return errors.Wrapf(err, "remove unmarked blob %s", digest) - } - n++ + if err := e.Walk(ctx, root, func(descriptor ispec.Descriptor) error { + seen[descriptor.Digest] = struct{}{} + return nil + }); err != nil { + return nil, err } - // Finally, tell CAS to GC it. - if err := engine.GC(ctx); err != nil { - return errors.Wrapf(err, "GC engine") + var reachable []digest.Digest + for node := range seen { + reachable = append(reachable, node) } - - log.Debugf("garbage collected %d blobs", n) - return nil + return reachable, nil } diff --git a/oci/config/convert/README.md b/oci/config/convert/README.md new file mode 100644 index 000000000..429161a5f --- /dev/null +++ b/oci/config/convert/README.md @@ -0,0 +1,11 @@ +### `umoci/oci/config/convert` ### + +One fairly important aspect of creating a runtime bundle is the configuration +of the container. While an image configuration and runtime configuration are +defined on different levels (images are far more platform agnostic than runtime +bundles), conversion from an image to a runtime configuration *is* (or rather +*will be*) defined as part of the OCI specification. + +This package implements a fairly unopinionated implementation of that +conversion, allowing consumers to easily add their own extensions in the +runtime configuration generation. diff --git a/oci/generate/runtime.go b/oci/config/convert/runtime.go similarity index 99% rename from oci/generate/runtime.go rename to oci/config/convert/runtime.go index 046e9e01e..53119f2ee 100644 --- a/oci/generate/runtime.go +++ b/oci/config/convert/runtime.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package generate +package convert import ( "path/filepath" diff --git a/oci/generate/README.md b/oci/config/generate/README.md similarity index 93% rename from oci/generate/README.md rename to oci/config/generate/README.md index bfb6c1104..ca3ac7340 100644 --- a/oci/generate/README.md +++ b/oci/config/generate/README.md @@ -1,4 +1,4 @@ -### `umoci/oci/generator` ### +### `umoci/oci/config/generate` ### This intends to be a library like `runtime-tools/generate` which allows you to generate modifications to an OCI image configuration blob (of type diff --git a/oci/generate/save.go b/oci/config/generate/save.go similarity index 100% rename from oci/generate/save.go rename to oci/config/generate/save.go diff --git a/oci/generate/spec.go b/oci/config/generate/spec.go similarity index 100% rename from oci/generate/spec.go rename to oci/config/generate/spec.go diff --git a/oci/generate/spec_test.go b/oci/config/generate/spec_test.go similarity index 100% rename from oci/generate/spec_test.go rename to oci/config/generate/spec_test.go diff --git a/oci/layer/unpack.go b/oci/layer/unpack.go index 07a690886..4ffe94e2f 100644 --- a/oci/layer/unpack.go +++ b/oci/layer/unpack.go @@ -30,7 +30,8 @@ import ( "github.com/apex/log" "github.com/openSUSE/umoci/oci/cas" - igen "github.com/openSUSE/umoci/oci/generate" + "github.com/openSUSE/umoci/oci/casext" + iconv "github.com/openSUSE/umoci/oci/config/convert" "github.com/openSUSE/umoci/pkg/idtools" "github.com/openSUSE/umoci/pkg/system" ispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -84,6 +85,8 @@ func isLayerType(mediaType string) bool { // // FIXME: This interface is ugly. func UnpackManifest(ctx context.Context, engine cas.Engine, bundle string, manifest ispec.Manifest, opt *MapOptions) error { + engineExt := casext.Engine{engine} + var mapOptions MapOptions if opt != nil { mapOptions = *opt @@ -143,7 +146,7 @@ func UnpackManifest(ctx context.Context, engine cas.Engine, bundle string, manif // In order to verify the DiffIDs as we extract layers, we have to get the // .Config blob first. But we can't extract it (generate the runtime // config) until after we have the full rootfs generated. - configBlob, err := cas.FromDescriptor(ctx, engine, manifest.Config) + configBlob, err := engineExt.FromDescriptor(ctx, manifest.Config) if err != nil { return errors.Wrap(err, "get config blob") } @@ -167,7 +170,7 @@ func UnpackManifest(ctx context.Context, engine cas.Engine, bundle string, manif layerDiffID := config.RootFS.DiffIDs[idx] log.Infof("unpack layer: %s", layerDescriptor.Digest) - layerBlob, err := cas.FromDescriptor(ctx, engine, layerDescriptor) + layerBlob, err := engineExt.FromDescriptor(ctx, layerDescriptor) if err != nil { return errors.Wrap(err, "get layer blob") } @@ -206,7 +209,7 @@ func UnpackManifest(ctx context.Context, engine cas.Engine, bundle string, manif log.Infof("unpack configuration: %s", configBlob.Digest) g := rgen.New() - if err := igen.MutateRuntimeSpec(g, rootfsPath, config, manifest); err != nil { + if err := iconv.MutateRuntimeSpec(g, rootfsPath, config, manifest); err != nil { return errors.Wrap(err, "generate config.json") }