feat(storage): Store blob content-type in extended attributes

After the discussion in #116, this stores the blob content types
in extended attributes when using the filesystem backend.

If the underlying filesystem doesn't support extended attributes,
storing blobs won't work; also, if extended attributes get removed,
blobs won't be served anymore. We can relax this behavior if
needed (i.e. log errors but still accept to store or serve blobs).
However, since the Docker Engine (and possibly other container
engines) won't accept to pull images from a registry that doesn't
use correct content types for manifest files, it could be argued
that it's better to give a hard fail. (Otherwise, the container
engine gives cryptic error messages like "missing signature key".)

I can change that behavior (and log errors but still store/serve
blobs to the filesystem) if you think it's better.

default.nix

@@ -38,7 +38,7 @@ let
     doCheck = true;
 
     # Needs to be updated after every modification of go.mod/go.sum
-    vendorSha256 = "1ff0kfww6fy6pnvyva7x8cc6l1d12aafps48wrkwawk2qjy9a8b9";
+    vendorSha256 = "1adjav0dxb97ws0w2k50rhk6r46wvfry6aj4sik3ninl525kd15s";
 
     buildFlagsArray = [
       "-ldflags=-s -w -X main.version=${nixery-commit-hash}"

go.mod

@@ -5,6 +5,7 @@ go 1.15
 require (
 	cloud.google.com/go/storage v1.15.0
 	github.com/google/go-cmp v0.5.5
+	github.com/pkg/xattr v0.4.3
 	github.com/sirupsen/logrus v1.8.1
 	golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c
 	gonum.org/v1/gonum v0.9.1

go.sum

@@ -158,6 +158,8 @@ github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2
 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/xattr v0.4.3 h1:5Jx4GCg5ABtqWZH8WLzeI4fOtM1HyX4RBawuCoua1es=
+github.com/pkg/xattr v0.4.3/go.mod h1:sBD3RAqlr8Q+RC3FutZcikpT8nyDrIEEBw2J744gVWs=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -322,6 +324,7 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

storage/filesystem.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"path"
 
+	"github.com/pkg/xattr"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -49,8 +50,7 @@ func (b *FSBackend) Name() string {
 	return fmt.Sprintf("Filesystem (%s)", b.path)
 }
 
-// TODO(tazjin): Implement support for persisting content-types for the filesystem backend.
-func (b *FSBackend) Persist(ctx context.Context, key, _type string, f Persister) (string, int64, error) {
+func (b *FSBackend) Persist(ctx context.Context, key, contentType string, f Persister) (string, int64, error) {
 	full := path.Join(b.path, key)
 	dir := path.Dir(full)
 	err := os.MkdirAll(dir, 0755)
@@ -66,6 +66,12 @@ func (b *FSBackend) Persist(ctx context.Context, key, _type string, f Persister)
 	}
 	defer file.Close()
 
+	err = xattr.Set(full, "user.mime_type", []byte(contentType))
+	if err != nil {
+		log.WithError(err).WithField("file", full).Error("failed to store file type in xattrs")
+		return "", 0, err
+	}
+
 	return f(file)
 }
 
@@ -92,6 +98,13 @@ func (b *FSBackend) Serve(digest string, r *http.Request, w http.ResponseWriter)
 		"path":   p,
 	}).Info("serving blob from filesystem")
 
+	contentType, err := xattr.Get(p, "user.mime_type")
+	if err != nil {
+		log.WithError(err).WithField("file", p).Error("failed to read file type from xattrs")
+		return err
+	}
+	w.Header().Add("Content-Type", string(contentType))
+
 	http.ServeFile(w, r, p)
 	return nil
 }