oci/*: major split ups

CHANGELOG.md

@@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
+### Changed
+- `umoci`'s `oci/cas` and `oci/config` libraries have been massively refactored
+  and rewritten, to allow for third-parties to use the OCI libraries. The plan
+  is for these to eventually become part of an OCI project. openSUSE/umoci#90
 
 ## [0.1.0] - 2017-02-11
 ### Added

cmd/umoci/config.go

@@ -24,7 +24,7 @@ import (
 	"github.com/apex/log"
 	"github.com/openSUSE/umoci/mutate"
 	"github.com/openSUSE/umoci/oci/cas"
-	igen "github.com/openSUSE/umoci/oci/generate"
+	igen "github.com/openSUSE/umoci/oci/config/generate"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"

cmd/umoci/gc.go

@@ -19,6 +19,7 @@ package main
 
 import (
 	"github.com/openSUSE/umoci/oci/cas"
+	"github.com/openSUSE/umoci/oci/casext"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
 	"golang.org/x/net/context"
@@ -56,8 +57,9 @@ func gc(ctx *cli.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "open CAS")
 	}
+	engineExt := casext.Engine{engine}
 	defer engine.Close()
 
 	// Run the GC.
-	return errors.Wrap(cas.GC(context.Background(), engine), "gc")
+	return errors.Wrap(engineExt.GC(context.Background()), "gc")
 }

cmd/umoci/init.go

@@ -54,7 +54,7 @@ func initLayout(ctx *cli.Context) error {
 		return errors.Wrap(err, "image layout creation")
 	}
 
-	if err := cas.CreateLayout(imagePath); err != nil {
+	if err := cas.Create(imagePath); err != nil {
 		return errors.Wrap(err, "image layout creation")
 	}
 

cmd/umoci/main.go

@@ -25,6 +25,9 @@ import (
 	logcli "github.com/apex/log/handlers/cli"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
+
+	// Include all official OCI images.
+	_ "github.com/openSUSE/umoci/oci/cas/drivers"
 )
 
 // version is version ID for the source, read from VERSION in the source and

cmd/umoci/new.go

@@ -23,7 +23,7 @@ import (
 
 	"github.com/apex/log"
 	"github.com/openSUSE/umoci/oci/cas"
-	igen "github.com/openSUSE/umoci/oci/generate"
+	igen "github.com/openSUSE/umoci/oci/config/generate"
 	imeta "github.com/opencontainers/image-spec/specs-go"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"

cmd/umoci/repack.go

@@ -28,7 +28,7 @@ import (
 	"github.com/openSUSE/umoci"
 	"github.com/openSUSE/umoci/mutate"
 	"github.com/openSUSE/umoci/oci/cas"
-	igen "github.com/openSUSE/umoci/oci/generate"
+	igen "github.com/openSUSE/umoci/oci/config/generate"
 	"github.com/openSUSE/umoci/oci/layer"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"

cmd/umoci/stat.go

@@ -23,6 +23,7 @@ import (
 	"os"
 
 	"github.com/openSUSE/umoci/oci/cas"
+	"github.com/openSUSE/umoci/oci/casext"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
@@ -63,6 +64,7 @@ func stat(ctx *cli.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "open CAS")
 	}
+	engineExt := casext.Engine{engine}
 	defer engine.Close()
 
 	manifestDescriptor, err := engine.GetReference(context.Background(), tagName)
@@ -76,7 +78,7 @@ func stat(ctx *cli.Context) error {
 	}
 
 	// Get stat information.
-	ms, err := Stat(context.Background(), engine, manifestDescriptor)
+	ms, err := Stat(context.Background(), engineExt, manifestDescriptor)
 	if err != nil {
 		return errors.Wrap(err, "stat")
 	}

cmd/umoci/unpack.go

@@ -26,6 +26,7 @@ import (
 	"github.com/apex/log"
 	"github.com/openSUSE/umoci"
 	"github.com/openSUSE/umoci/oci/cas"
+	"github.com/openSUSE/umoci/oci/casext"
 	"github.com/openSUSE/umoci/oci/layer"
 	"github.com/openSUSE/umoci/pkg/idtools"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -125,15 +126,16 @@ func unpack(ctx *cli.Context) error {
 	if err != nil {
 		return errors.Wrap(err, "open CAS")
 	}
+	engineExt := casext.Engine{engine}
 	defer engine.Close()
 
-	fromDescriptor, err := engine.GetReference(context.Background(), fromName)
+	fromDescriptor, err := engineExt.GetReference(context.Background(), fromName)
 	if err != nil {
 		return errors.Wrap(err, "get descriptor")
 	}
 	meta.From = fromDescriptor
 
-	manifestBlob, err := cas.FromDescriptor(context.Background(), engine, meta.From)
+	manifestBlob, err := engineExt.FromDescriptor(context.Background(), meta.From)
 	if err != nil {
 		return errors.Wrap(err, "get manifest")
 	}
@@ -172,7 +174,7 @@ func unpack(ctx *cli.Context) error {
 	//        should be fixed once the CAS engine PR is merged into
 	//        image-tools. https://github.com/opencontainers/image-tools/pull/5
 	log.Info("unpacking bundle ...")
-	if err := layer.UnpackManifest(context.Background(), engine, bundlePath, manifest, &meta.MapOptions); err != nil {
+	if err := layer.UnpackManifest(context.Background(), engineExt, bundlePath, manifest, &meta.MapOptions); err != nil {
 		return errors.Wrap(err, "create runtime bundle")
 	}
 	log.Info("... done")

cmd/umoci/utils.go

@@ -28,8 +28,8 @@ import (
 	"text/tabwriter"
 
 	"github.com/docker/go-units"
-	"github.com/openSUSE/umoci/oci/cas"
-	igen "github.com/openSUSE/umoci/oci/generate"
+	"github.com/openSUSE/umoci/oci/casext"
+	igen "github.com/openSUSE/umoci/oci/config/generate"
 	"github.com/openSUSE/umoci/oci/layer"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -180,15 +180,15 @@ type historyStat struct {
 
 // Stat computes the ManifestStat for a given manifest blob. The provided
 // descriptor must refer to an OCI Manifest.
-func Stat(ctx context.Context, engine cas.Engine, manifestDescriptor ispec.Descriptor) (ManifestStat, error) {
+func Stat(ctx context.Context, engine casext.Engine, manifestDescriptor ispec.Descriptor) (ManifestStat, error) {
 	var stat ManifestStat
 
 	if manifestDescriptor.MediaType != ispec.MediaTypeImageManifest {
 		return stat, errors.Errorf("stat: cannot stat a non-manifest descriptor: invalid media type '%s'", manifestDescriptor.MediaType)
 	}
 
 	// We have to get the actual manifest.
-	manifestBlob, err := cas.FromDescriptor(ctx, engine, manifestDescriptor)
+	manifestBlob, err := engine.FromDescriptor(ctx, manifestDescriptor)
 	if err != nil {
 		return stat, err
 	}
@@ -199,7 +199,7 @@ func Stat(ctx context.Context, engine cas.Engine, manifestDescriptor ispec.Descr
 	}
 
 	// Now get the config.
-	configBlob, err := cas.FromDescriptor(ctx, engine, manifest.Config)
+	configBlob, err := engine.FromDescriptor(ctx, manifest.Config)
 	if err != nil {
 		return stat, errors.Wrap(err, "stat")
 	}

mutate/mutate.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/openSUSE/umoci/oci/cas"
+	"github.com/openSUSE/umoci/oci/casext"
 	"github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -40,7 +41,7 @@ func manifestPtr(m ispec.Manifest) *ispec.Manifest { return &m }
 // TODO: Implement manifest list support.
 type Mutator struct {
 	// These are the arguments we got in New().
-	engine cas.Engine
+	engine casext.Engine
 	source ispec.Descriptor
 
 	// Cached values of the configuration and manifest.
@@ -75,7 +76,7 @@ type Meta struct {
 func (m *Mutator) cache(ctx context.Context) error {
 	// We need the manifest
 	if m.manifest == nil {
-		blob, err := cas.FromDescriptor(ctx, m.engine, m.source)
+		blob, err := m.engine.FromDescriptor(ctx, m.source)
 		if err != nil {
 			return errors.Wrap(err, "cache source manifest")
 		}
@@ -92,7 +93,7 @@ func (m *Mutator) cache(ctx context.Context) error {
 	}
 
 	if m.config == nil {
-		blob, err := cas.FromDescriptor(ctx, m.engine, m.manifest.Config)
+		blob, err := m.engine.FromDescriptor(ctx, m.manifest.Config)
 		if err != nil {
 			return errors.Wrap(err, "cache source config")
 		}
@@ -120,7 +121,7 @@ func New(engine cas.Engine, src ispec.Descriptor) (*Mutator, error) {
 	}
 
 	return &Mutator{
-		engine: engine,
+		engine: casext.Engine{engine},
 		source: src,
 	}, nil
 }

mutate/mutate_test.go

@@ -30,6 +30,9 @@ import (
 	imeta "github.com/opencontainers/image-spec/specs-go"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"golang.org/x/net/context"
+
+	// Include all known drivers.
+	_ "github.com/openSUSE/umoci/oci/cas/drivers"
 )
 
 // These come from just running the code.
@@ -41,7 +44,7 @@ const (
 
 func setup(t *testing.T, dir string) (cas.Engine, ispec.Descriptor) {
 	dir = filepath.Join(dir, "image")
-	if err := cas.CreateLayout(dir); err != nil {
+	if err := cas.Create(dir); err != nil {
 		t.Fatal(err)
 	}
 

oci/cas/cas.go

@@ -20,8 +20,6 @@ package cas
 import (
 	"fmt"
 	"io"
-	"os"
-	"path/filepath"
 
 	// We need to include sha256 in order for go-digest to properly handle such
 	// hashes, since Go's crypto library like to lazy-load cryptographic
@@ -30,24 +28,13 @@ import (
 
 	"github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 )
 
 const (
 	// BlobAlgorithm is the name of the only supported digest algorithm for blobs.
 	// FIXME: We can make this a list.
 	BlobAlgorithm = digest.SHA256
-
-	// refDirectory is the directory inside an OCI image that contains references.
-	refDirectory = "refs"
-
-	// blobDirectory is the directory inside an OCI image that contains blobs.
-	blobDirectory = "blobs"
-
-	// layoutFile is the file in side an OCI image the indicates what version
-	// of the OCI spec the image is.
-	layoutFile = "oci-layout"
 )
 
 // Exposed errors.
@@ -77,6 +64,11 @@ type Engine interface {
 	// as the reader. Note that due to intricacies in the Go JSON
 	// implementation, we cannot guarantee that two calls to PutBlobJSON() will
 	// return the same digest.
+	//
+	// TODO: Use a proper JSON serialisation library, which actually guarantees
+	//       consistent output. Go's JSON library doesn't even attempt to sort
+	//       map[...]... objects (which have their iteration order randomised
+	//       in Go).
 	PutBlobJSON(ctx context.Context, data interface{}) (digest digest.Digest, size int64, err error)
 
 	// PutReference adds a new reference descriptor blob to the image. This is
@@ -110,52 +102,13 @@ type Engine interface {
 	// ListReferences returns the set of reference names stored in the image.
 	ListReferences(ctx context.Context) (names []string, err error)
 
-	// GC executes a garbage collection of any non-blob garbage in the store
+	// Clean executes a garbage collection of any non-blob garbage in the store
 	// (this includes temporary files and directories not reachable from the
 	// CAS interface). This MUST NOT remove any blobs or references in the
 	// store.
-	GC(ctx context.Context) (err error)
+	Clean(ctx context.Context) (err error)
 
 	// Close releases all references held by the engine. Subsequent operations
 	// may fail.
 	Close() (err error)
 }
-
-// Open will create an Engine reference to the OCI image at the provided
-// path. If the image format is not supported, ErrNotImplemented will be
-// returned. If the path does not exist, os.ErrNotExist will be returned.
-func Open(path string) (Engine, error) {
-	fi, err := os.Stat(path)
-	if err != nil {
-		return nil, err
-	}
-
-	if fi.IsDir() {
-		return newDirEngine(path)
-	}
-
-	return nil, ErrNotImplemented
-}
-
-// blobPath returns the path to a blob given its digest, relative to the root
-// of the OCI image. The digest must be of the form algorithm:hex.
-func blobPath(digest digest.Digest) (string, error) {
-	if err := digest.Validate(); err != nil {
-		return "", errors.Wrapf(err, "invalid digest: %q", digest)
-	}
-
-	algo := digest.Algorithm()
-	hash := digest.Hex()
-
-	if algo != BlobAlgorithm {
-		return "", errors.Errorf("unsupported algorithm: %q", algo)
-	}
-
-	return filepath.Join(blobDirectory, algo.String(), hash), nil
-}
-
-// refPath returns the path to a reference given its name, relative to the r
-// oot of the OCI image.
-func refPath(name string) (string, error) {
-	return filepath.Join(refDirectory, name), nil
-}