Skip to content

*: add ability to use a shared CAS directory #190

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jonboulle wants to merge 3 commits into from
Closed

*: add ability to use a shared CAS directory #190

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
62945df
*: add ability to use a shared CAS directory
jonboulle Oct 11, 2017
a9347c5
cmd/gc: error out if --shared-cas is supplied
jonboulle Oct 24, 2017
d5e1a58
test: add {re,un}pack test for --shared-cas
jonboulle Oct 24, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cmd/umoci/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ func parseKV(input string) (string, string, error) {
func config(ctx *cli.Context) error {
imagePath := ctx.App.Metadata["--image-path"].(string)
fromName := ctx.App.Metadata["--image-tag"].(string)
sharedCasPath := ctx.String("shared-cas")

// By default we clobber the old tag.
tagName := fromName
Expand All @@ -131,7 +132,7 @@ func config(ctx *cli.Context) error {
}

// Get a reference to the CAS.
engine, err := dir.Open(imagePath)
engine, err := dir.Open(imagePath, sharedCasPath)
if err != nil {
return errors.Wrap(err, "open CAS")
}
Expand Down
5 changes: 4 additions & 1 deletion cmd/umoci/gc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,12 @@ root set of references. All other blobs will be removed.`,

func gc(ctx *cli.Context) error {
imagePath := ctx.App.Metadata["--image-path"].(string)
if ctx.String("shared-cas") != "" {
return errors.New("gc not supported with --shared-cas")
}

// Get a reference to the CAS.
engine, err := dir.Open(imagePath)
engine, err := dir.Open(imagePath, "")
if err != nil {
return errors.Wrap(err, "open CAS")
}
Expand Down
3 changes: 2 additions & 1 deletion cmd/umoci/new.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,10 @@ needing a base image to start from.`,
func newImage(ctx *cli.Context) error {
imagePath := ctx.App.Metadata["--image-path"].(string)
tagName := ctx.App.Metadata["--image-tag"].(string)
sharedCasPath := ctx.String("shared-cas")

// Get a reference to the CAS.
engine, err := dir.Open(imagePath)
engine, err := dir.Open(imagePath, sharedCasPath)
if err != nil {
return errors.Wrap(err, "open CAS")
}
Expand Down
3 changes: 2 additions & 1 deletion cmd/umoci/raw-runtime-config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ func rawConfig(ctx *cli.Context) error {
imagePath := ctx.App.Metadata["--image-path"].(string)
fromName := ctx.App.Metadata["--image-tag"].(string)
configPath := ctx.App.Metadata["config"].(string)
sharedCasPath := ctx.String("shared-cas")

var meta UmociMeta
meta.Version = UmociMetaVersion
Expand Down Expand Up @@ -124,7 +125,7 @@ func rawConfig(ctx *cli.Context) error {
}).Debugf("parsed mappings")

// Get a reference to the CAS.
engine, err := dir.Open(imagePath)
engine, err := dir.Open(imagePath, sharedCasPath)
if err != nil {
return errors.Wrap(err, "open CAS")
}
Expand Down
3 changes: 2 additions & 1 deletion cmd/umoci/repack.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ func repack(ctx *cli.Context) error {
imagePath := ctx.App.Metadata["--image-path"].(string)
tagName := ctx.App.Metadata["--image-tag"].(string)
bundlePath := ctx.App.Metadata["bundle"].(string)
sharedCasPath := ctx.String("shared-cas")

// Read the metadata first.
meta, err := ReadBundleMeta(bundlePath)
Expand All @@ -117,7 +118,7 @@ func repack(ctx *cli.Context) error {
}

// Get a reference to the CAS.
engine, err := dir.Open(imagePath)
engine, err := dir.Open(imagePath, sharedCasPath)
if err != nil {
return errors.Wrap(err, "open CAS")
}
Expand Down
3 changes: 2 additions & 1 deletion cmd/umoci/stat.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,10 @@ humans to read, and might change in future versions.`,
func stat(ctx *cli.Context) error {
imagePath := ctx.App.Metadata["--image-path"].(string)
tagName := ctx.App.Metadata["--image-tag"].(string)
sharedCasPath := ctx.String("shared-cas")

// Get a reference to the CAS.
engine, err := dir.Open(imagePath)
engine, err := dir.Open(imagePath, sharedCasPath)
if err != nil {
return errors.Wrap(err, "open CAS")
}
Expand Down
9 changes: 6 additions & 3 deletions cmd/umoci/tag.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,10 @@ func tagAdd(ctx *cli.Context) error {
imagePath := ctx.App.Metadata["--image-path"].(string)
fromName := ctx.App.Metadata["--image-tag"].(string)
tagName := ctx.App.Metadata["new-tag"].(string)
sharedCasPath := ctx.String("shared-cas")

// Get a reference to the CAS.
engine, err := dir.Open(imagePath)
engine, err := dir.Open(imagePath, sharedCasPath)
if err != nil {
return errors.Wrap(err, "open CAS")
}
Expand Down Expand Up @@ -111,9 +112,10 @@ tag to remove.`,
func tagRemove(ctx *cli.Context) error {
imagePath := ctx.App.Metadata["--image-path"].(string)
tagName := ctx.App.Metadata["--image-tag"].(string)
sharedCasPath := ctx.String("shared-cas")

// Get a reference to the CAS.
engine, err := dir.Open(imagePath)
engine, err := dir.Open(imagePath, sharedCasPath)
if err != nil {
return errors.Wrap(err, "open CAS")
}
Expand Down Expand Up @@ -148,9 +150,10 @@ line. See umoci-stat(1) to get more information about each tagged image.`,

func tagList(ctx *cli.Context) error {
imagePath := ctx.App.Metadata["--image-path"].(string)
sharedCasPath := ctx.String("shared-cas")

// Get a reference to the CAS.
engine, err := dir.Open(imagePath)
engine, err := dir.Open(imagePath, sharedCasPath)
if err != nil {
return errors.Wrap(err, "open CAS")
}
Expand Down
3 changes: 2 additions & 1 deletion cmd/umoci/unpack.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ func unpack(ctx *cli.Context) error {
imagePath := ctx.App.Metadata["--image-path"].(string)
fromName := ctx.App.Metadata["--image-tag"].(string)
bundlePath := ctx.App.Metadata["bundle"].(string)
sharedCasPath := ctx.String("shared-cas")

var meta UmociMeta
meta.Version = UmociMetaVersion
Expand Down Expand Up @@ -120,7 +121,7 @@ func unpack(ctx *cli.Context) error {
}).Debugf("parsed mappings")

// Get a reference to the CAS.
engine, err := dir.Open(imagePath)
engine, err := dir.Open(imagePath, sharedCasPath)
if err != nil {
return errors.Wrap(err, "open CAS")
}
Expand Down
16 changes: 15 additions & 1 deletion cmd/umoci/utils_ux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,12 +130,19 @@ func uxTag(cmd cli.Command) cli.Command {
// tag) will be stored in ctx.Metadata["--image-path"] and
// ctx.Metadata["--image-tag"] as strings (both will be nil if --image is not
// specified).
// A --shared-cas flag is also added to the given cli.Command, which allows
// users to provide a different directory to be used for blob operations
func uxImage(cmd cli.Command) cli.Command {
cmd.Flags = append(cmd.Flags, cli.StringFlag{
Name: "image",
Usage: "OCI image URI of the form 'path[:tag]'",
})

cmd.Flags = append(cmd.Flags, cli.StringFlag{
Name: "shared-cas",
Usage: "shared directory to use for blobs (instead of internal image layout directory)",
})

oldBefore := cmd.Before
cmd.Before = func(ctx *cli.Context) error {
// Verify and parse --image.
Expand Down Expand Up @@ -181,15 +188,22 @@ func uxImage(cmd cli.Command) cli.Command {
return cmd
}

// uxLayout adds an --layout flag to the given cli.Command as well as adding
// uxLayout adds a --layout flag to the given cli.Command as well as adding
// relevant validation logic to the .Before of the command. The value is stored
// in ctx.App.Metadata["--image-path"] as a string (or nil --layout was not set).
// A --shared-cas flag is also added to the given cli.Command, which allows
// users to provide a different directory to be used for blob operations
func uxLayout(cmd cli.Command) cli.Command {
cmd.Flags = append(cmd.Flags, cli.StringFlag{
Name: "layout",
Usage: "path to an OCI image layout",
})

cmd.Flags = append(cmd.Flags, cli.StringFlag{
Name: "shared-cas",
Usage: "shared directory to use for blobs (instead of internal image layout directory)",
})

oldBefore := cmd.Before
cmd.Before = func(ctx *cli.Context) error {
// Verify and parse --layout.
Expand Down
2 changes: 1 addition & 1 deletion mutate/mutate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ func (m *Mutator) add(ctx context.Context, reader io.Reader) (digest.Digest, int
return "", -1, errors.Wrap(err, "getting cache failed")
}

diffidDigester := cas.BlobAlgorithm.Digester()
diffidDigester := cas.DefaultBlobAlgorithm.Digester()
hashReader := io.TeeReader(reader, diffidDigester.Hash())

pipeReader, pipeWriter := io.Pipe()
Expand Down
8 changes: 4 additions & 4 deletions mutate/mutate_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,12 @@ const (
)

func setup(t *testing.T, dir string) (cas.Engine, ispec.Descriptor) {
dir = filepath.Join(dir, "image")
if err := casdir.Create(dir); err != nil {
image := filepath.Join(dir, "image")
if err := casdir.Create(image); err != nil {
t.Fatal(err)
}

engine, err := casdir.Open(dir)
engine, err := casdir.Open(image, "")
if err != nil {
t.Fatal(err)
}
Expand All @@ -71,7 +71,7 @@ func setup(t *testing.T, dir string) (cas.Engine, ispec.Descriptor) {
tw.Close()

// Push the base layer.
diffidDigester := cas.BlobAlgorithm.Digester()
diffidDigester := cas.DefaultBlobAlgorithm.Digester()
hashReader := io.TeeReader(&buffer, diffidDigester.Hash())
layerDigest, layerSize, err := engine.PutBlob(context.Background(), hashReader)
if err != nil {
Expand Down
13 changes: 9 additions & 4 deletions oci/cas/cas.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,18 @@ import (
)

const (
// BlobAlgorithm is the name of the only supported digest algorithm for blobs.
// FIXME: We can make this a list.
BlobAlgorithm = digest.SHA256
// DefaultBlobAlgorithm is the default supported digest algorithm for blobs
DefaultBlobAlgorithm = digest.SHA256
Copy link
Contributor

@wking wking Oct 18, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deprecate in favor of go-digest's Canonical?

Copy link
Contributor

@wking wking Oct 18, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deprecate in favor of go-digest's Canonical?

Spun off into #197.

)

// Exposed errors.
var (
// BlobAlgorithms contains the supported digest algorithms for blobs
BlobAlgorithms = []digest.Algorithm{
digest.SHA256,
}

// Exposed errors

// ErrNotExist is effectively an implementation-neutral version of
// os.ErrNotExist.
ErrNotExist = fmt.Errorf("no such blob or index")
Expand Down
22 changes: 11 additions & 11 deletions oci/cas/dir/cas_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func TestCreateLayout(t *testing.T) {
t.Fatalf("unexpected error creating image: %+v", err)
}

engine, err := Open(image)
engine, err := Open(image, "")
if err != nil {
t.Fatalf("unexpected error opening image: %+v", err)
}
Expand Down Expand Up @@ -85,7 +85,7 @@ func TestEngineBlob(t *testing.T) {
t.Fatalf("unexpected error creating image: %+v", err)
}

engine, err := Open(image)
engine, err := Open(image, "")
if err != nil {
t.Fatalf("unexpected error opening image: %+v", err)
}
Expand All @@ -98,7 +98,7 @@ func TestEngineBlob(t *testing.T) {
{[]byte("some blob")},
{[]byte("another blob")},
} {
digester := cas.BlobAlgorithm.Digester()
digester := cas.DefaultBlobAlgorithm.Digester()
if _, err := io.Copy(digester.Hash(), bytes.NewReader(test.bytes)); err != nil {
t.Fatalf("could not hash bytes: %+v", err)
}
Expand Down Expand Up @@ -172,7 +172,7 @@ func TestEngineValidate(t *testing.T) {
if err != nil {
t.Fatal(err)
}
engine, err = Open(image)
engine, err = Open(image, "")
if err == nil {
t.Errorf("expected to get an error")
engine.Close()
Expand All @@ -186,7 +186,7 @@ func TestEngineValidate(t *testing.T) {
if err := ioutil.WriteFile(filepath.Join(image, layoutFile), []byte("invalid JSON"), 0644); err != nil {
t.Fatal(err)
}
engine, err = Open(image)
engine, err = Open(image, "")
if err == nil {
t.Errorf("expected to get an error")
engine.Close()
Expand All @@ -200,7 +200,7 @@ func TestEngineValidate(t *testing.T) {
if err := ioutil.WriteFile(filepath.Join(image, layoutFile), []byte("{}"), 0644); err != nil {
t.Fatal(err)
}
engine, err = Open(image)
engine, err = Open(image, "")
if err == nil {
t.Errorf("expected to get an error")
engine.Close()
Expand All @@ -220,7 +220,7 @@ func TestEngineValidate(t *testing.T) {
if err := os.RemoveAll(filepath.Join(image, blobDirectory)); err != nil {
t.Fatalf("unexpected error deleting blobdir: %+v", err)
}
engine, err = Open(image)
engine, err = Open(image, "")
if err == nil {
t.Errorf("expected to get an error")
engine.Close()
Expand All @@ -243,7 +243,7 @@ func TestEngineValidate(t *testing.T) {
if err := ioutil.WriteFile(filepath.Join(image, blobDirectory), []byte(""), 0755); err != nil {
t.Fatal(err)
}
engine, err = Open(image)
engine, err = Open(image, "")
if err == nil {
t.Errorf("expected to get an error")
engine.Close()
Expand All @@ -263,7 +263,7 @@ func TestEngineValidate(t *testing.T) {
if err := os.RemoveAll(filepath.Join(image, indexFile)); err != nil {
t.Fatalf("unexpected error deleting index: %+v", err)
}
engine, err = Open(image)
engine, err = Open(image, "")
if err == nil {
t.Errorf("expected to get an error")
engine.Close()
Expand All @@ -286,15 +286,15 @@ func TestEngineValidate(t *testing.T) {
if err := os.Mkdir(filepath.Join(image, indexFile), 0755); err != nil {
t.Fatal(err)
}
engine, err = Open(image)
engine, err = Open(image, "")
if err == nil {
t.Errorf("expected to get an error")
engine.Close()
}

// No such directory.
image = filepath.Join(root, "non-exist")
engine, err = Open(image)
engine, err = Open(image, "")
if err == nil {
t.Errorf("expected to get an error")
engine.Close()
Expand Down
Loading