From c7e5d145f99ddbafaf0188e537692c699031e683 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 9 Jan 2024 10:43:13 -0500 Subject: [PATCH 01/40] Implement fsverity functionality Implement the necessary calls to fsverity for enabling fsverity on a file, measuring an fsverity file, and checking if fsverity is enabled on a file. Signed-off-by: James Jenkins --- pkg/fsverity/fsverity_linux.go | 108 +++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 pkg/fsverity/fsverity_linux.go diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go new file mode 100644 index 000000000000..cda1f85e5ea5 --- /dev/null +++ b/pkg/fsverity/fsverity_linux.go @@ -0,0 +1,108 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package fsverity + +import ( + "fmt" + "os" + "syscall" + "unsafe" + + "golang.org/x/sys/unix" +) + +type fsverityEnableArg struct { + version uint32 + hash_algorithm uint32 + block_size uint32 + salt_size uint32 + salt_ptr uint64 + sig_size uint32 + reserved1 uint32 + sig_ptr uint64 + reserved2 [11]uint64 +} + +type fsverityDigest struct { + digest_algorithm uint16 + digest_size uint16 + digest [64]uint8 +} + +const maxDigestSize int = 64 + +func IsEnabled(path string) (bool, error) { + f, err := os.OpenFile(path, os.O_RDONLY, 0644) + if err != nil { + return false, fmt.Errorf("Error opening file: %s", err) + } + + var attr int + + _, _, flagErr := unix.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(unix.FS_IOC_GETFLAGS), uintptr(unsafe.Pointer(&attr))) + if flagErr != 0 { + return false, fmt.Errorf("Error getting inode flags: %s", flagErr) + } + + if attr&unix.FS_VERITY_FL == unix.FS_VERITY_FL { + return true, nil + } + + return false, fmt.Errorf("File %s is not a verity file", path) +} + +func Enable(path string) (bool, error) { + f, err := os.OpenFile(path, os.O_RDONLY, 0644) + if err != nil { + return false, fmt.Errorf("Error opening file: %s\n", err.Error()) + } + + var args *fsverityEnableArg = &fsverityEnableArg{} + args.version = 1 + args.hash_algorithm = 1 + // TODO: change block size based on the system page size + args.block_size = 65536 + + _, _, errno := unix.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(unix.FS_IOC_ENABLE_VERITY), uintptr(unsafe.Pointer(args))) + if errno != 0 { + return false, fmt.Errorf("Enable fsverity failed: %d\n", errno) + } + + return true, nil +} + +func Measure(path string) (string, error) { + var verityDigest string + f, err := os.OpenFile(path, os.O_RDONLY, 0644) + if err != nil { + fmt.Printf("Error opening file: %s\n", err.Error()) + return + } + + var d *fsverityDigest = &fsverityDigest{digest_size: maxDigestSize} + _, _, errno := unix.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(unix.FS_IOC_MEASURE_VERITY), uintptr(unsafe.Pointer(d))) + if errno != 0 { + return verityDigest, fmt.Errorf("Measure fsverity failed: %d\n", errno) + } + + var i uint16 + for i = 0; i < (*d).digest_size; i++ { + verityDigest = fmt.Sprintf("%s%x", verityDigest, (*d).digest[i]) + } + + return verityDigest, nil +} From 5c9db9455a58013bda987ef746165bf6ba1a50fa Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 9 Jan 2024 16:23:02 -0500 Subject: [PATCH 02/40] Set proper fsverity block size Set fsverity block size equal to the system page size when possible. Signed-off-by: James Jenkins --- pkg/fsverity/fsverity_linux.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index cda1f85e5ea5..92a07503958f 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -43,7 +43,10 @@ type fsverityDigest struct { digest [64]uint8 } -const maxDigestSize int = 64 +const ( + maxDigestSize int = 64 + defaultBlockSize int = 4096 +) func IsEnabled(path string) (bool, error) { f, err := os.OpenFile(path, os.O_RDONLY, 0644) @@ -74,8 +77,12 @@ func Enable(path string) (bool, error) { var args *fsverityEnableArg = &fsverityEnableArg{} args.version = 1 args.hash_algorithm = 1 - // TODO: change block size based on the system page size - args.block_size = 65536 + + blockSize := unix.Getpagesize() + if blockSize <= 0 { + blockSize = defaultBlockSize + } + args.block_size = blockSize _, _, errno := unix.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(unix.FS_IOC_ENABLE_VERITY), uintptr(unsafe.Pointer(args))) if errno != 0 { From 7dc5829ba64d4a3ebf7ed2eee6ff7df9fcdadd8f Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Fri, 12 Jan 2024 10:44:17 -0500 Subject: [PATCH 03/40] Clean up fsverity implementation Modify enable function return values and clean up calls to open the target file. --- pkg/fsverity/fsverity_linux.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index 92a07503958f..2b3cc93363b6 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -49,7 +49,7 @@ const ( ) func IsEnabled(path string) (bool, error) { - f, err := os.OpenFile(path, os.O_RDONLY, 0644) + f, err := os.Open(path) if err != nil { return false, fmt.Errorf("Error opening file: %s", err) } @@ -65,13 +65,13 @@ func IsEnabled(path string) (bool, error) { return true, nil } - return false, fmt.Errorf("File %s is not a verity file", path) + return false, nil } -func Enable(path string) (bool, error) { - f, err := os.OpenFile(path, os.O_RDONLY, 0644) +func Enable(path string) error { + f, err := os.Open(path) if err != nil { - return false, fmt.Errorf("Error opening file: %s\n", err.Error()) + return fmt.Errorf("Error opening file: %s\n", err.Error()) } var args *fsverityEnableArg = &fsverityEnableArg{} @@ -86,15 +86,15 @@ func Enable(path string) (bool, error) { _, _, errno := unix.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(unix.FS_IOC_ENABLE_VERITY), uintptr(unsafe.Pointer(args))) if errno != 0 { - return false, fmt.Errorf("Enable fsverity failed: %d\n", errno) + return fmt.Errorf("Enable fsverity failed: %d\n", errno) } - return true, nil + return nil } func Measure(path string) (string, error) { var verityDigest string - f, err := os.OpenFile(path, os.O_RDONLY, 0644) + f, err := os.Open(path) if err != nil { fmt.Printf("Error opening file: %s\n", err.Error()) return From 7644a95ba9d5fff7e002ab0996cafc1ade141f73 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Fri, 12 Jan 2024 10:46:22 -0500 Subject: [PATCH 04/40] Integrate fsveriy Integrate fsverity with the local content ingester and provider. Ingester enables fsverity on content blobs when blobs are written to disk and the provider checks the verity digest when the blobs are read. --- content/local/store.go | 29 +++++++++++++++++++++++++++++ content/local/writer.go | 16 ++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/content/local/store.go b/content/local/store.go index 7af3d2ae9230..2d6750d8c11f 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -22,6 +22,7 @@ import ( "io" "os" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -128,6 +129,34 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. return nil, fmt.Errorf("calculating blob path for ReaderAt: %w", err) } + if runtime.GOOS == "linux" { + // check that fsverity is enabled on the blob before reading + // if not, it may not be trustworthy + enabled, err := fsverity.IsEnabled(p) + if err != nil { + log.G(ctx).WithError(err).Errorf("Error checking fsverity status: %s", p) + } + if !ok { + log.G(ctx).Warnf("fsverity not enabled on blob %s", p) + } else { + verityDigest, merr := fsverity.Measure(target) + if merr != nil { + log.G(ctx).WithField("ref", w.ref).Error("failed to take fsverity measurement of blob") + } else { + // compare the digest to the "good" value stored in the blob label + blobInfo, err := s.Info(ctx, desc.Digest) + if err != nil { + log.G(ctx).Error("failed to retrieve good fsverity digest from store") + } else { + if verityDigest != blobInfo.Labels["fsverity_digest"] { + log.G(ctx).Error("fsverity digest does not match known good value") + return nil, fmt.Errorf("blob is not trusted, fsverity digest failed verification") + } + } + } + } + } + reader, err := OpenReader(p) if err != nil { return nil, fmt.Errorf("blob %s expected at %s: %w", desc.Digest, p, err) diff --git a/content/local/writer.go b/content/local/writer.go index d22c3365cd67..639855df4396 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -137,6 +137,22 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, return err } + if runtime.GOOS == "linux" { + // Enable fsverity digest verification on the blob + if err := fsverity.Enable(target); err != nil { + log.G(ctx).WithField("ref", w.ref).Error("failed to enable fsverity verification") + } else { + verityDigest, merr := fsverity.Measure(target) + if merr != nil { + log.G(ctx).WithField("ref", w.ref).Error("failed to take fsverity measurement of blob") + } else { + // store the fsverity digest for later comparison + // TODO: create a better label for the fs verity digest + base.Labels["fsverity_digest"] = verityDigest + } + } + } + // Ingest has now been made available in the content store, attempt to complete // setting metadata but errors should only be logged and not returned since // the content store cannot be cleanly rolled back. From 5438a7364cd4cdf21828dfb2dfc1ca39386a5f2f Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Fri, 19 Jan 2024 12:02:47 -0500 Subject: [PATCH 05/40] Add fsverity debug messages --- content/local/store.go | 2 ++ content/local/writer.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/content/local/store.go b/content/local/store.go index 2d6750d8c11f..3c6d2e981674 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -130,6 +130,7 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. } if runtime.GOOS == "linux" { + log.G(ctx).Debugf("verifying blob with fsverity") // check that fsverity is enabled on the blob before reading // if not, it may not be trustworthy enabled, err := fsverity.IsEnabled(p) @@ -143,6 +144,7 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. if merr != nil { log.G(ctx).WithField("ref", w.ref).Error("failed to take fsverity measurement of blob") } else { + log.G(ctx).Debugf("comparing measured digest to known good value") // compare the digest to the "good" value stored in the blob label blobInfo, err := s.Info(ctx, desc.Digest) if err != nil { diff --git a/content/local/writer.go b/content/local/writer.go index 639855df4396..e249b64e79b3 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -138,6 +138,7 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, } if runtime.GOOS == "linux" { + log.G(ctx).Debugf("enabling fsverity on blob") // Enable fsverity digest verification on the blob if err := fsverity.Enable(target); err != nil { log.G(ctx).WithField("ref", w.ref).Error("failed to enable fsverity verification") @@ -146,6 +147,7 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, if merr != nil { log.G(ctx).WithField("ref", w.ref).Error("failed to take fsverity measurement of blob") } else { + log.G(ctx).Debugf("storing \"good\" digest value in metadata database") // store the fsverity digest for later comparison // TODO: create a better label for the fs verity digest base.Labels["fsverity_digest"] = verityDigest From d178237f3910aa79067ff25351a3ae868b388962 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Mon, 22 Jan 2024 16:25:31 -0500 Subject: [PATCH 06/40] Add fsverity package imports --- content/local/store.go | 1 + content/local/writer.go | 1 + 2 files changed, 2 insertions(+) diff --git a/content/local/store.go b/content/local/store.go index 3c6d2e981674..e3e97cbd2b37 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -31,6 +31,7 @@ import ( "github.com/containerd/containerd/v2/content" "github.com/containerd/containerd/v2/errdefs" "github.com/containerd/containerd/v2/filters" + "github.com/containerd/containerd/v2/pkg/fsverity" "github.com/containerd/log" "github.com/opencontainers/go-digest" diff --git a/content/local/writer.go b/content/local/writer.go index e249b64e79b3..134d8383a702 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -28,6 +28,7 @@ import ( "github.com/containerd/containerd/v2/content" "github.com/containerd/containerd/v2/errdefs" + "github.com/containerd/containerd/v2/pkg/fsverity" "github.com/containerd/log" "github.com/opencontainers/go-digest" ) From 3d453ae7fdf85cb4410024a5199cbbed08d7af77 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 23 Jan 2024 10:36:46 -0500 Subject: [PATCH 07/40] Fix errors in fsverity package --- pkg/fsverity/fsverity_linux.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index 2b3cc93363b6..b62ef30542dc 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -44,8 +44,8 @@ type fsverityDigest struct { } const ( - maxDigestSize int = 64 - defaultBlockSize int = 4096 + maxDigestSize uint16 = 64 + defaultBlockSize uint32 = 4096 ) func IsEnabled(path string) (bool, error) { @@ -96,8 +96,7 @@ func Measure(path string) (string, error) { var verityDigest string f, err := os.Open(path) if err != nil { - fmt.Printf("Error opening file: %s\n", err.Error()) - return + return verityDigest, fmt.Errorf("Error opening file: %s\n", err.Error()) } var d *fsverityDigest = &fsverityDigest{digest_size: maxDigestSize} From bbe117e4a576ee5148d0826a318610c45319f283 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 23 Jan 2024 10:45:40 -0500 Subject: [PATCH 08/40] Fix types of constants when assinging to structs --- pkg/fsverity/fsverity_linux.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index b62ef30542dc..40c4d5e7ba66 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -44,8 +44,8 @@ type fsverityDigest struct { } const ( - maxDigestSize uint16 = 64 - defaultBlockSize uint32 = 4096 + maxDigestSize int = 64 + defaultBlockSize int = 4096 ) func IsEnabled(path string) (bool, error) { @@ -82,7 +82,7 @@ func Enable(path string) error { if blockSize <= 0 { blockSize = defaultBlockSize } - args.block_size = blockSize + args.block_size = uint32(blockSize) _, _, errno := unix.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(unix.FS_IOC_ENABLE_VERITY), uintptr(unsafe.Pointer(args))) if errno != 0 { From 39187901ea15a1bce664d6310bb9da44b60315aa Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 23 Jan 2024 10:47:56 -0500 Subject: [PATCH 09/40] Fix type of maxDigestSize constant --- pkg/fsverity/fsverity_linux.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index 40c4d5e7ba66..4faf1f0b480e 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -44,8 +44,8 @@ type fsverityDigest struct { } const ( - maxDigestSize int = 64 - defaultBlockSize int = 4096 + defaultBlockSize int = 4096 + maxDigestSize uint16 = 64 ) func IsEnabled(path string) (bool, error) { From 8bf3c2a97007d7893b0e1182551eed91e6ee6639 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 23 Jan 2024 10:52:05 -0500 Subject: [PATCH 10/40] Fix errors in variable names Fix errors in variable names that were left unchanged when copying logic from another file. --- content/local/store.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index e3e97cbd2b37..ac3cdeba024d 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -138,12 +138,12 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. if err != nil { log.G(ctx).WithError(err).Errorf("Error checking fsverity status: %s", p) } - if !ok { + if !enabled { log.G(ctx).Warnf("fsverity not enabled on blob %s", p) } else { - verityDigest, merr := fsverity.Measure(target) + verityDigest, merr := fsverity.Measure(p) if merr != nil { - log.G(ctx).WithField("ref", w.ref).Error("failed to take fsverity measurement of blob") + log.G(ctx).WithField("blob", p).Error("failed to take fsverity measurement of blob") } else { log.G(ctx).Debugf("comparing measured digest to known good value") // compare the digest to the "good" value stored in the blob label From 0c1023db0e0429e3e664565073c036bbebc8d0db Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 23 Jan 2024 12:29:47 -0500 Subject: [PATCH 11/40] Add more details to fsverity logs for debugging --- content/local/store.go | 8 ++++---- content/local/writer.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index ac3cdeba024d..e83d00add161 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -136,23 +136,23 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. // if not, it may not be trustworthy enabled, err := fsverity.IsEnabled(p) if err != nil { - log.G(ctx).WithError(err).Errorf("Error checking fsverity status: %s", p) + log.G(ctx).WithError(err).Errorf("Error checking fsverity status of blob %s: %s", p, err.Error()) } if !enabled { log.G(ctx).Warnf("fsverity not enabled on blob %s", p) } else { verityDigest, merr := fsverity.Measure(p) if merr != nil { - log.G(ctx).WithField("blob", p).Error("failed to take fsverity measurement of blob") + log.G(ctx).WithField("blob", p).Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) } else { log.G(ctx).Debugf("comparing measured digest to known good value") // compare the digest to the "good" value stored in the blob label blobInfo, err := s.Info(ctx, desc.Digest) if err != nil { - log.G(ctx).Error("failed to retrieve good fsverity digest from store") + log.G(ctx).Errorf("failed to retrieve good fsverity digest from store: %s", err.Error()) } else { if verityDigest != blobInfo.Labels["fsverity_digest"] { - log.G(ctx).Error("fsverity digest does not match known good value") + log.G(ctx).Errorf("fsverity digest does not match known good value, expected: %s; got: %s", blobInfo.Labels["fsverity_digest"], verityDigest) return nil, fmt.Errorf("blob is not trusted, fsverity digest failed verification") } } diff --git a/content/local/writer.go b/content/local/writer.go index 134d8383a702..6742775cb745 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -142,11 +142,11 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, log.G(ctx).Debugf("enabling fsverity on blob") // Enable fsverity digest verification on the blob if err := fsverity.Enable(target); err != nil { - log.G(ctx).WithField("ref", w.ref).Error("failed to enable fsverity verification") + log.G(ctx).WithField("ref", w.ref).Errorf("failed to enable fsverity verification: %s", err.Error()) } else { verityDigest, merr := fsverity.Measure(target) if merr != nil { - log.G(ctx).WithField("ref", w.ref).Error("failed to take fsverity measurement of blob") + log.G(ctx).WithField("ref", w.ref).Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) } else { log.G(ctx).Debugf("storing \"good\" digest value in metadata database") // store the fsverity digest for later comparison From 5b13b9af9911060ea7745f7183e48743bc22cf40 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 23 Jan 2024 15:09:44 -0500 Subject: [PATCH 12/40] Set fsverity block size correctly fsverity block size should be set to the minimum of the system page size and the file system block size. --- pkg/fsverity/fsverity_linux.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index 4faf1f0b480e..47e029133246 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -78,10 +78,21 @@ func Enable(path string) error { args.version = 1 args.hash_algorithm = 1 + // fsverity block size should be the minimum between the page size + // and the file system block size + // If neither value is retrieved successfully, set fsverity block size to the default value blockSize := unix.Getpagesize() + + s := unix.Stat_t{} + serr := unix.Stat(path, &s) + if serr == nil && int(s.BlkSize) < blockSize { + blockSize = int(s.BlkSize) + } + if blockSize <= 0 { blockSize = defaultBlockSize } + args.block_size = uint32(blockSize) _, _, errno := unix.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(unix.FS_IOC_ENABLE_VERITY), uintptr(unsafe.Pointer(args))) From c1e16fd777cb5a8cce2bd54f4521aa2f8e387d59 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 23 Jan 2024 15:19:02 -0500 Subject: [PATCH 13/40] Fix block size field name --- pkg/fsverity/fsverity_linux.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index 47e029133246..5638b9cbc205 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -85,8 +85,8 @@ func Enable(path string) error { s := unix.Stat_t{} serr := unix.Stat(path, &s) - if serr == nil && int(s.BlkSize) < blockSize { - blockSize = int(s.BlkSize) + if serr == nil && int(s.Blksize) < blockSize { + blockSize = int(s.Blksize) } if blockSize <= 0 { From 333e54fba926281fcda60d225574a44543376c71 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 23 Jan 2024 15:47:07 -0500 Subject: [PATCH 14/40] Fix nil map error --- content/local/writer.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/content/local/writer.go b/content/local/writer.go index 6742775cb745..8fc4a05c799e 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -151,6 +151,9 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, log.G(ctx).Debugf("storing \"good\" digest value in metadata database") // store the fsverity digest for later comparison // TODO: create a better label for the fs verity digest + if base.Labels == nil { + base.Labels = make(map[string]string) + } base.Labels["fsverity_digest"] = verityDigest } } From 4abca35ea4c6da5d29b4e42239420028db2c2277 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 23 Jan 2024 16:14:39 -0500 Subject: [PATCH 15/40] Add blob details to debug logs --- content/local/store.go | 3 +++ content/local/writer.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/content/local/store.go b/content/local/store.go index e83d00add161..bd3cd12f7bbe 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -130,6 +130,8 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. return nil, fmt.Errorf("calculating blob path for ReaderAt: %w", err) } + log.G(ctx).Debugf("Getting reader for blob %v", p) + if runtime.GOOS == "linux" { log.G(ctx).Debugf("verifying blob with fsverity") // check that fsverity is enabled on the blob before reading @@ -153,6 +155,7 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. } else { if verityDigest != blobInfo.Labels["fsverity_digest"] { log.G(ctx).Errorf("fsverity digest does not match known good value, expected: %s; got: %s", blobInfo.Labels["fsverity_digest"], verityDigest) + log.G(ctx).Debugf("blob labels: %v", blobInfo.Labels) return nil, fmt.Errorf("blob is not trusted, fsverity digest failed verification") } } diff --git a/content/local/writer.go b/content/local/writer.go index 8fc4a05c799e..a27b005cdf75 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -139,7 +139,7 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, } if runtime.GOOS == "linux" { - log.G(ctx).Debugf("enabling fsverity on blob") + log.G(ctx).Debugf("enabling fsverity on blob %v", target) // Enable fsverity digest verification on the blob if err := fsverity.Enable(target); err != nil { log.G(ctx).WithField("ref", w.ref).Errorf("failed to enable fsverity verification: %s", err.Error()) From a1487ed5a3b18e4210d239a4ae434c5b0aceed86 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 23 Jan 2024 16:22:28 -0500 Subject: [PATCH 16/40] Check if labels are getting properly added --- content/local/writer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/content/local/writer.go b/content/local/writer.go index a27b005cdf75..907332516eb2 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -174,6 +174,7 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, } if w.s.ls != nil && base.Labels != nil { + log.G(ctx).Debugf("content labels: %v", base.Labels) if err := w.s.ls.Set(dgst, base.Labels); err != nil { log.G(ctx).WithField("digest", dgst).Error("failed to set labels") } From 91af4a2bbea178957caafd1a0e3bbccdf9d8c621 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 23 Jan 2024 16:25:44 -0500 Subject: [PATCH 17/40] Change position of log for more detail --- content/local/writer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/local/writer.go b/content/local/writer.go index 907332516eb2..cb2333ee2194 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -173,8 +173,8 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, log.G(ctx).WithField("ref", w.ref).WithField("path", w.path).Error("failed to remove ingest directory") } + log.G(ctx).Debugf("content labels: %v", base.Labels) if w.s.ls != nil && base.Labels != nil { - log.G(ctx).Debugf("content labels: %v", base.Labels) if err := w.s.ls.Set(dgst, base.Labels); err != nil { log.G(ctx).WithField("digest", dgst).Error("failed to set labels") } From 737649cfeb9b4ade395cff31acf2ed08bc423a9f Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Fri, 26 Jan 2024 09:29:50 -0500 Subject: [PATCH 18/40] Add TODO comments for improvements --- content/local/writer.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/content/local/writer.go b/content/local/writer.go index cb2333ee2194..f472f2da4e86 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -149,8 +149,14 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, log.G(ctx).WithField("ref", w.ref).Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) } else { log.G(ctx).Debugf("storing \"good\" digest value in metadata database") + // store the fsverity digest for later comparison + // + // TODO: find how to add content labels in the metadata database + // the 'content store' is not able to store labels on its own + // // TODO: create a better label for the fs verity digest + // TODO: This does not work, fix it (see comment above) if base.Labels == nil { base.Labels = make(map[string]string) } From f4eb7288cfa13514f6bdcbd7e5887685822d3a09 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Mon, 29 Jan 2024 14:55:20 -0500 Subject: [PATCH 19/40] Store fsverity digest values Store fsverity digest values in integrity files on the file system with the blob data. --- content/local/store.go | 17 ++++++++++------- content/local/writer.go | 40 ++++++++++++++++++++++++++++++---------- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index bd3cd12f7bbe..3777a83ac6cb 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -148,16 +148,19 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. log.G(ctx).WithField("blob", p).Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) } else { log.G(ctx).Debugf("comparing measured digest to known good value") + // compare the digest to the "good" value stored in the blob label - blobInfo, err := s.Info(ctx, desc.Digest) + var expectedDigest string + integrityFile := filepath.Join(s.root, "integrity", desc.Digest.Encoded()) + b, err := io.ReadAll(integrityFile) if err != nil { - log.G(ctx).Errorf("failed to retrieve good fsverity digest from store: %s", err.Error()) + log.G(ctx).Errorf("could not read fsverity digest from integrity file: %s", err.Error()) } else { - if verityDigest != blobInfo.Labels["fsverity_digest"] { - log.G(ctx).Errorf("fsverity digest does not match known good value, expected: %s; got: %s", blobInfo.Labels["fsverity_digest"], verityDigest) - log.G(ctx).Debugf("blob labels: %v", blobInfo.Labels) - return nil, fmt.Errorf("blob is not trusted, fsverity digest failed verification") - } + expectedDigest = string(b) + } + if verityDigest != expectedDigest { + log.G(ctx).Errorf("Error: fsverity digest does not match the expected digest") + return nil, fmt.Errorf("blob not trusted: fsverity digest does not match the expected digest value") } } } diff --git a/content/local/writer.go b/content/local/writer.go index f472f2da4e86..d0210a113123 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -150,17 +150,37 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, } else { log.G(ctx).Debugf("storing \"good\" digest value in metadata database") - // store the fsverity digest for later comparison - // - // TODO: find how to add content labels in the metadata database - // the 'content store' is not able to store labels on its own - // - // TODO: create a better label for the fs verity digest - // TODO: This does not work, fix it (see comment above) - if base.Labels == nil { - base.Labels = make(map[string]string) + integrityStore := filepath.Join(w.s.root, "integrity") + if err := os.MkdirAll(integrityStore, 0755); err != nil { + log.G(ctx).Debugf("error creating integrity digest directory: %s", err.Error()) + return err } - base.Labels["fsverity_digest"] = verityDigest + + digestPath := filepath.Join(integrityStore, dgst.Encoded()) + digestInfo, err := os.Stat(digestPath) + if err != nil { + if os.IsExist(err) { + log.G(ctx).Debugf("integrity digest for blob already exists") + return err + } + } + + digestFile, err := os.Create(digestPath) + if err != nil { + if os.IsExist(err) { + log.G(ctx).Debugf("Error creating integrity digest file: digest for blob already exists") + return err + } + + return fmt.Errorf("Error creating integrity digest file for blob: %s", dgst.Encoded()) + } + + _, err := digestFile.WriteString(verityDigest) + if err != nil { + log.G(ctx).Debugf("Error writing fsverity digest to file: %s", err) + return err + } + } } } From 458b8a99171a341c2a686b736ba661f9bdff3c0f Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Mon, 29 Jan 2024 15:17:00 -0500 Subject: [PATCH 20/40] Fix errors Get a file descriptor to read integrity file and fix variable assignments. --- content/local/store.go | 7 ++++++- content/local/writer.go | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index 3777a83ac6cb..4a4715663dd1 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -152,7 +152,12 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. // compare the digest to the "good" value stored in the blob label var expectedDigest string integrityFile := filepath.Join(s.root, "integrity", desc.Digest.Encoded()) - b, err := io.ReadAll(integrityFile) + ifd, err := os.Open(integrityFile) + if err != nil { + log.G(ctx).Errorf("failed to read integrity file of blob %s", p) + return nil, fmt.Errorf("could not read expected integrity value of %s", p) + } + b, err := io.ReadAll(ifd) if err != nil { log.G(ctx).Errorf("could not read fsverity digest from integrity file: %s", err.Error()) } else { diff --git a/content/local/writer.go b/content/local/writer.go index d0210a113123..2962c2a4dd2a 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -157,7 +157,7 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, } digestPath := filepath.Join(integrityStore, dgst.Encoded()) - digestInfo, err := os.Stat(digestPath) + _, err := os.Stat(digestPath) if err != nil { if os.IsExist(err) { log.G(ctx).Debugf("integrity digest for blob already exists") @@ -175,7 +175,7 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, return fmt.Errorf("Error creating integrity digest file for blob: %s", dgst.Encoded()) } - _, err := digestFile.WriteString(verityDigest) + _, err = digestFile.WriteString(verityDigest) if err != nil { log.G(ctx).Debugf("Error writing fsverity digest to file: %s", err) return err From bf5e0e58adbc8f2517da39c36ad768d148e8caea Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 30 Jan 2024 12:02:19 -0500 Subject: [PATCH 21/40] Remove integrity digests when the blob is deleted --- content/local/store.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/content/local/store.go b/content/local/store.go index 4a4715663dd1..cfd408c31592 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -197,6 +197,15 @@ func (s *store) Delete(ctx context.Context, dgst digest.Digest) error { return fmt.Errorf("content %v: %w", dgst, errdefs.ErrNotFound) } + integrityFile := filepath.Join(s.root, "integrity", desc.Digest.Encoded()) + if err := os.RemoveAll(integrityFile); err != nil { + if !osIsNotExist(err) { + return err + } + + return fmt.Errorf("integrity file %v: %w", dgst, errdefs.ErrNotFound) + } + return nil } From 8aec398dc595cffd86c6aeff735615b703701ff0 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 30 Jan 2024 12:06:22 -0500 Subject: [PATCH 22/40] Fix typos --- content/local/store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index cfd408c31592..189d0944f262 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -197,9 +197,9 @@ func (s *store) Delete(ctx context.Context, dgst digest.Digest) error { return fmt.Errorf("content %v: %w", dgst, errdefs.ErrNotFound) } - integrityFile := filepath.Join(s.root, "integrity", desc.Digest.Encoded()) + integrityFile := filepath.Join(s.root, "integrity", dgst.Encoded()) if err := os.RemoveAll(integrityFile); err != nil { - if !osIsNotExist(err) { + if !os.IsNotExist(err) { return err } From 4c2f56e1a411b04d0b3bb1d4a36d1891fcd67d57 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 30 Jan 2024 15:03:51 -0500 Subject: [PATCH 23/40] Add local function Begin adding local function to improve logic readability and flow. --- content/local/store.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/content/local/store.go b/content/local/store.go index 189d0944f262..bfdc1ed25e40 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -131,9 +131,13 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. } log.G(ctx).Debugf("Getting reader for blob %v", p) + measure := func() { + log.G(ctx).Debugf("measuring blob: %s", p) + } if runtime.GOOS == "linux" { log.G(ctx).Debugf("verifying blob with fsverity") + measure() // check that fsverity is enabled on the blob before reading // if not, it may not be trustworthy enabled, err := fsverity.IsEnabled(p) From 7f37729f7ef77ccc435d4528f040f6869fe67735 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 30 Jan 2024 16:01:07 -0500 Subject: [PATCH 24/40] Refactor ReaderAt Refactor ReaderAt to improve readability of fsverity validation logic. --- content/local/store.go | 66 ++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index bfdc1ed25e40..d01285c9ad24 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -131,46 +131,50 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. } log.G(ctx).Debugf("Getting reader for blob %v", p) - measure := func() { + measure := func() (string, error) { + var verityDigest string log.G(ctx).Debugf("measuring blob: %s", p) - } - - if runtime.GOOS == "linux" { - log.G(ctx).Debugf("verifying blob with fsverity") - measure() // check that fsverity is enabled on the blob before reading // if not, it may not be trustworthy enabled, err := fsverity.IsEnabled(p) if err != nil { - log.G(ctx).WithError(err).Errorf("Error checking fsverity status of blob %s: %s", p, err.Error()) + return verityDigest, fmt.Errorf("Error checking fsverity status of blob %s: %s", p, err.Error()) } if !enabled { - log.G(ctx).Warnf("fsverity not enabled on blob %s", p) + return verityDigest, fmt.Errorf("fsverity not enabled on blob %s", p) + } + verityDigest, merr := fsverity.Measure(p) + if merr != nil { + return verityDigest, fmt.Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) + } + return verityDigest, nil + } + + if runtime.GOOS == "linux" { + log.G(ctx).Debugf("verifying blob with fsverity") + verityDigest, err := measure() + if err != nil { + log.G(ctx).WithField("blob", p).Errorf("failed to get integrity measurement: %s", err.Error()) } else { - verityDigest, merr := fsverity.Measure(p) - if merr != nil { - log.G(ctx).WithField("blob", p).Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) + var expectedDigest string + integrityFile := filepath.Join(s.root, "integrity", desc.Digest.Encoded()) + ifd, err := os.Open(integrityFile) + if err != nil { + log.G(ctx).Errorf("failed to read integrity file of blob %s", p) + return nil, fmt.Errorf("could not read expected integrity value of %s", p) + } + b, err := io.ReadAll(ifd) + if err != nil { + log.G(ctx).Errorf("could not read fsverity digest from integrity file: %s", err.Error()) } else { - log.G(ctx).Debugf("comparing measured digest to known good value") - - // compare the digest to the "good" value stored in the blob label - var expectedDigest string - integrityFile := filepath.Join(s.root, "integrity", desc.Digest.Encoded()) - ifd, err := os.Open(integrityFile) - if err != nil { - log.G(ctx).Errorf("failed to read integrity file of blob %s", p) - return nil, fmt.Errorf("could not read expected integrity value of %s", p) - } - b, err := io.ReadAll(ifd) - if err != nil { - log.G(ctx).Errorf("could not read fsverity digest from integrity file: %s", err.Error()) - } else { - expectedDigest = string(b) - } - if verityDigest != expectedDigest { - log.G(ctx).Errorf("Error: fsverity digest does not match the expected digest") - return nil, fmt.Errorf("blob not trusted: fsverity digest does not match the expected digest value") - } + expectedDigest = string(b) + } + + // compare the digest to the "good" value stored in the blob label + log.G(ctx).Debugf("comparing measured digest to known good value") + if verityDigest != expectedDigest { + log.G(ctx).Errorf("Error: fsverity digest does not match the expected digest") + return nil, fmt.Errorf("blob not trusted: fsverity digest does not match the expected digest value") } } } From 15dc96e089f1c1e2d933269cf99373796da1e109 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 30 Jan 2024 16:57:51 -0500 Subject: [PATCH 25/40] Refactor content writer Refactor content writer to make fsverity verification logic more readable. --- content/local/writer.go | 72 +++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/content/local/writer.go b/content/local/writer.go index 2962c2a4dd2a..0eea004f0849 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -139,48 +139,56 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, } if runtime.GOOS == "linux" { - log.G(ctx).Debugf("enabling fsverity on blob %v", target) - // Enable fsverity digest verification on the blob - if err := fsverity.Enable(target); err != nil { - log.G(ctx).WithField("ref", w.ref).Errorf("failed to enable fsverity verification: %s", err.Error()) - } else { + enable := func() (string, error) { + var verityDigest string + log.G(ctx).Debugf("enabling fsverity on blob %v", target) + // Enable fsverity digest verification on the blob + if err := fsverity.Enable(target); err != nil { + return verityDigest, fmt.Errorf("failed to enable fsverity verification: %s", err.Error()) + } + verityDigest, merr := fsverity.Measure(target) if merr != nil { - log.G(ctx).WithField("ref", w.ref).Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) - } else { - log.G(ctx).Debugf("storing \"good\" digest value in metadata database") - - integrityStore := filepath.Join(w.s.root, "integrity") - if err := os.MkdirAll(integrityStore, 0755); err != nil { - log.G(ctx).Debugf("error creating integrity digest directory: %s", err.Error()) - return err - } + return verityDigest, fmt.Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) + } + return verityDigest, nil + } - digestPath := filepath.Join(integrityStore, dgst.Encoded()) - _, err := os.Stat(digestPath) - if err != nil { - if os.IsExist(err) { - log.G(ctx).Debugf("integrity digest for blob already exists") - return err - } - } + verityDigest, err := enable() + if err != nil { + log.G(ctx).Errorf("enabling fsverity on blob failed: %s", err.Error()) + } else { + log.G(ctx).Debugf("storing \"good\" digest value in metadata database") - digestFile, err := os.Create(digestPath) - if err != nil { - if os.IsExist(err) { - log.G(ctx).Debugf("Error creating integrity digest file: digest for blob already exists") - return err - } + integrityStore := filepath.Join(w.s.root, "integrity") + if err := os.MkdirAll(integrityStore, 0755); err != nil { + log.G(ctx).Debugf("error creating integrity digest directory: %s", err.Error()) + return err + } - return fmt.Errorf("Error creating integrity digest file for blob: %s", dgst.Encoded()) + digestPath := filepath.Join(integrityStore, dgst.Encoded()) + _, err := os.Stat(digestPath) + if err != nil { + if os.IsExist(err) { + log.G(ctx).Debugf("integrity digest for blob already exists") + return err } + } - _, err = digestFile.WriteString(verityDigest) - if err != nil { - log.G(ctx).Debugf("Error writing fsverity digest to file: %s", err) + digestFile, err := os.Create(digestPath) + if err != nil { + if os.IsExist(err) { + log.G(ctx).Debugf("Error creating integrity digest file: digest for blob already exists") return err } + return fmt.Errorf("Error creating integrity digest file for blob: %s", dgst.Encoded()) + } + + _, err = digestFile.WriteString(verityDigest) + if err != nil { + log.G(ctx).Debugf("Error writing fsverity digest to file: %s", err) + return err } } } From ddaa0f290b12e5aa439f75cdc3a081d6e0b77632 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Wed, 31 Jan 2024 10:55:47 -0500 Subject: [PATCH 26/40] Move measure function Move measure function so that it is only initialized when it is needed (on linux systems only). --- content/local/store.go | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index d01285c9ad24..75b32ed5b9ec 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -131,26 +131,27 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. } log.G(ctx).Debugf("Getting reader for blob %v", p) - measure := func() (string, error) { - var verityDigest string - log.G(ctx).Debugf("measuring blob: %s", p) - // check that fsverity is enabled on the blob before reading - // if not, it may not be trustworthy - enabled, err := fsverity.IsEnabled(p) - if err != nil { - return verityDigest, fmt.Errorf("Error checking fsverity status of blob %s: %s", p, err.Error()) - } - if !enabled { - return verityDigest, fmt.Errorf("fsverity not enabled on blob %s", p) - } - verityDigest, merr := fsverity.Measure(p) - if merr != nil { - return verityDigest, fmt.Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) - } - return verityDigest, nil - } if runtime.GOOS == "linux" { + measure := func() (string, error) { + var verityDigest string + log.G(ctx).Debugf("measuring blob: %s", p) + // check that fsverity is enabled on the blob before reading + // if not, it may not be trustworthy + enabled, err := fsverity.IsEnabled(p) + if err != nil { + return verityDigest, fmt.Errorf("Error checking fsverity status of blob %s: %s", p, err.Error()) + } + if !enabled { + return verityDigest, fmt.Errorf("fsverity not enabled on blob %s", p) + } + verityDigest, merr := fsverity.Measure(p) + if merr != nil { + return verityDigest, fmt.Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) + } + return verityDigest, nil + } + log.G(ctx).Debugf("verifying blob with fsverity") verityDigest, err := measure() if err != nil { From a52deba5744c9410aaafaa992858a74ee257c632 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Fri, 9 Feb 2024 14:27:28 -0500 Subject: [PATCH 27/40] Create fsverity IsSupported function --- content/local/writer.go | 2 +- pkg/fsverity/fsverity_linux.go | 6 ++++++ pkg/fsverity/fsverity_other.go | 7 +++++++ 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 pkg/fsverity/fsverity_other.go diff --git a/content/local/writer.go b/content/local/writer.go index 0eea004f0849..eb7ab6f8d586 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -138,7 +138,7 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, return err } - if runtime.GOOS == "linux" { + if fsverity.IsSupported() { enable := func() (string, error) { var verityDigest string log.G(ctx).Debugf("enabling fsverity on blob %v", target) diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index 5638b9cbc205..e7a8b1850504 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -1,3 +1,5 @@ +//go:build linux + /* Copyright The containerd Authors. @@ -48,6 +50,10 @@ const ( maxDigestSize uint16 = 64 ) +func IsSupported() bool { + return true +} + func IsEnabled(path string) (bool, error) { f, err := os.Open(path) if err != nil { diff --git a/pkg/fsverity/fsverity_other.go b/pkg/fsverity/fsverity_other.go new file mode 100644 index 000000000000..c6c2e1a47b9d --- /dev/null +++ b/pkg/fsverity/fsverity_other.go @@ -0,0 +1,7 @@ +//go:build !linux + +package fsverity + +func IsSupported() bool { + return false +} From dbb0b8bd3ce2acf49231b1e229fd324bdc45eff6 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Fri, 9 Feb 2024 15:48:32 -0500 Subject: [PATCH 28/40] Use IsSupported function in content store --- content/local/store.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index 75b32ed5b9ec..d4a1e0ccf529 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -22,7 +22,6 @@ import ( "io" "os" "path/filepath" - "runtime" "strconv" "strings" "sync" @@ -132,7 +131,7 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. log.G(ctx).Debugf("Getting reader for blob %v", p) - if runtime.GOOS == "linux" { + if fsverity.IsSupported() { measure := func() (string, error) { var verityDigest string log.G(ctx).Debugf("measuring blob: %s", p) From d1157e705a9375a3962acb3f8e5e37c61670e759 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Mon, 12 Feb 2024 12:26:59 -0500 Subject: [PATCH 29/40] Remove fsverity IsEnabled check Fsverity measure function returns an error on its own if fsverity is not enabled on a file, no need for a redundant check. --- content/local/store.go | 10 +--------- content/local/writer.go | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index d4a1e0ccf529..6099cc466ba5 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -131,19 +131,11 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. log.G(ctx).Debugf("Getting reader for blob %v", p) + // validate the integrity of the blob if integrity validation is supported if fsverity.IsSupported() { measure := func() (string, error) { var verityDigest string log.G(ctx).Debugf("measuring blob: %s", p) - // check that fsverity is enabled on the blob before reading - // if not, it may not be trustworthy - enabled, err := fsverity.IsEnabled(p) - if err != nil { - return verityDigest, fmt.Errorf("Error checking fsverity status of blob %s: %s", p, err.Error()) - } - if !enabled { - return verityDigest, fmt.Errorf("fsverity not enabled on blob %s", p) - } verityDigest, merr := fsverity.Measure(p) if merr != nil { return verityDigest, fmt.Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) diff --git a/content/local/writer.go b/content/local/writer.go index eb7ab6f8d586..98f81aed9ac2 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -158,7 +158,7 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, if err != nil { log.G(ctx).Errorf("enabling fsverity on blob failed: %s", err.Error()) } else { - log.G(ctx).Debugf("storing \"good\" digest value in metadata database") + log.G(ctx).Debugf("storing \"good\" digest value") integrityStore := filepath.Join(w.s.root, "integrity") if err := os.MkdirAll(integrityStore, 0755); err != nil { From ae4a24c41dd04ba0433dd1f1ef7565943b55fbbd Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Mon, 12 Feb 2024 13:05:07 -0500 Subject: [PATCH 30/40] Separate integrity logic Separate logic of storing the integrity value of a blob into its own function. Makes code easier to read. --- content/local/store.go | 80 ++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index 6099cc466ba5..304205544d31 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -130,45 +130,8 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. } log.G(ctx).Debugf("Getting reader for blob %v", p) - - // validate the integrity of the blob if integrity validation is supported - if fsverity.IsSupported() { - measure := func() (string, error) { - var verityDigest string - log.G(ctx).Debugf("measuring blob: %s", p) - verityDigest, merr := fsverity.Measure(p) - if merr != nil { - return verityDigest, fmt.Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) - } - return verityDigest, nil - } - - log.G(ctx).Debugf("verifying blob with fsverity") - verityDigest, err := measure() - if err != nil { - log.G(ctx).WithField("blob", p).Errorf("failed to get integrity measurement: %s", err.Error()) - } else { - var expectedDigest string - integrityFile := filepath.Join(s.root, "integrity", desc.Digest.Encoded()) - ifd, err := os.Open(integrityFile) - if err != nil { - log.G(ctx).Errorf("failed to read integrity file of blob %s", p) - return nil, fmt.Errorf("could not read expected integrity value of %s", p) - } - b, err := io.ReadAll(ifd) - if err != nil { - log.G(ctx).Errorf("could not read fsverity digest from integrity file: %s", err.Error()) - } else { - expectedDigest = string(b) - } - - // compare the digest to the "good" value stored in the blob label - log.G(ctx).Debugf("comparing measured digest to known good value") - if verityDigest != expectedDigest { - log.G(ctx).Errorf("Error: fsverity digest does not match the expected digest") - return nil, fmt.Errorf("blob not trusted: fsverity digest does not match the expected digest value") - } - } + if err = storeIntegrity(s.root, p, desc); err != nil { + log.G(ctx).Errorf("error storing integrity value of blob %v: %s", p, err) } reader, err := OpenReader(p) @@ -736,3 +699,42 @@ func writeToCompletion(path string, data []byte, mode os.FileMode) error { } return nil } + +func storeIntegrity(rootPath string, p string, desc ocispec.Descriptor) error { + // validate the integrity of the blob if integrity validation is supported + if !fsverity.IsSupported() { + return fmt.Errorf("integrity validation is not supported") + } + + log.G(ctx).Debugf("verifying blob with fsverity") + + var verityDigest string + log.G(ctx).Debugf("measuring blob: %s", p) + verityDigest, merr := fsverity.Measure(p) + if merr != nil { + return fmt.Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) + } + + var expectedDigest string + integrityFile := filepath.Join(rootPath, "integrity", desc.Digest.Encoded()) + ifd, err := os.Open(integrityFile) + if err != nil { + log.G(ctx).Errorf("failed to read integrity file of blob %s", p) + return fmt.Errorf("could not read expected integrity value of %s", p) + } + b, err := io.ReadAll(ifd) + if err != nil { + log.G(ctx).Errorf("could not read fsverity digest from integrity file: %s", err.Error()) + } else { + expectedDigest = string(b) + } + + // compare the digest to the "good" value stored in the blob label + log.G(ctx).Debugf("comparing measured digest to known good value") + if verityDigest != expectedDigest { + log.G(ctx).Errorf("Error: fsverity digest does not match the expected digest") + return fmt.Errorf("blob not trusted: fsverity digest does not match the expected digest value") + } + + return nil +} From d6f90741e48c968652359fa2b5f652d97df0dc9b Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Mon, 12 Feb 2024 14:47:28 -0500 Subject: [PATCH 31/40] Separate integrity validation logic Separate integrity validation logic into a new function. Makes the code easier to read. --- content/local/store.go | 2 +- content/local/writer.go | 107 ++++++++++++++++++++-------------------- 2 files changed, 55 insertions(+), 54 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index 304205544d31..ae60a4dc9488 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -131,7 +131,7 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. log.G(ctx).Debugf("Getting reader for blob %v", p) if err = storeIntegrity(s.root, p, desc); err != nil { - log.G(ctx).Errorf("error storing integrity value of blob %v: %s", p, err) + log.G(ctx).Errorf("error storing integrity value of blob %v: %s", p, err.Error()) } reader, err := OpenReader(p) diff --git a/content/local/writer.go b/content/local/writer.go index 98f81aed9ac2..9ed2901c5d56 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -138,59 +138,8 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, return err } - if fsverity.IsSupported() { - enable := func() (string, error) { - var verityDigest string - log.G(ctx).Debugf("enabling fsverity on blob %v", target) - // Enable fsverity digest verification on the blob - if err := fsverity.Enable(target); err != nil { - return verityDigest, fmt.Errorf("failed to enable fsverity verification: %s", err.Error()) - } - - verityDigest, merr := fsverity.Measure(target) - if merr != nil { - return verityDigest, fmt.Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) - } - return verityDigest, nil - } - - verityDigest, err := enable() - if err != nil { - log.G(ctx).Errorf("enabling fsverity on blob failed: %s", err.Error()) - } else { - log.G(ctx).Debugf("storing \"good\" digest value") - - integrityStore := filepath.Join(w.s.root, "integrity") - if err := os.MkdirAll(integrityStore, 0755); err != nil { - log.G(ctx).Debugf("error creating integrity digest directory: %s", err.Error()) - return err - } - - digestPath := filepath.Join(integrityStore, dgst.Encoded()) - _, err := os.Stat(digestPath) - if err != nil { - if os.IsExist(err) { - log.G(ctx).Debugf("integrity digest for blob already exists") - return err - } - } - - digestFile, err := os.Create(digestPath) - if err != nil { - if os.IsExist(err) { - log.G(ctx).Debugf("Error creating integrity digest file: digest for blob already exists") - return err - } - - return fmt.Errorf("Error creating integrity digest file for blob: %s", dgst.Encoded()) - } - - _, err = digestFile.WriteString(verityDigest) - if err != nil { - log.G(ctx).Debugf("Error writing fsverity digest to file: %s", err) - return err - } - } + if err = validateIntegrity(w.s.root, target, dgst); err != nil { + log.G(ctx).Errorf("failed to validate integrity of blob %v: %s", target, err.Error()) } // Ingest has now been made available in the content store, attempt to complete @@ -263,3 +212,55 @@ func (w *writer) Truncate(size int64) error { } return w.fp.Truncate(0) } + +func validateIntegrity(rootPath string, target string, dgst digest.Digest) error { + if !fsverity.IsSupported() { + return fmt.Errorf("integrity validation is not supported") + } + + var verityDigest string + log.G(ctx).Debugf("enabling fsverity on blob %v", target) + if err := fsverity.Enable(target); err != nil { + return fmt.Errorf("failed to enable fsverity verification: %s", err.Error()) + } + + verityDigest, merr := fsverity.Measure(target) + if merr != nil { + return fmt.Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) + } + + log.G(ctx).Debugf("storing \"good\" digest value") + + integrityStore := filepath.Join(rootPath, "integrity") + if err := os.MkdirAll(integrityStore, 0755); err != nil { + log.G(ctx).Debugf("error creating integrity digest directory: %s", err.Error()) + return err + } + + digestPath := filepath.Join(integrityStore, dgst.Encoded()) + _, err := os.Stat(digestPath) + if err != nil { + if os.IsExist(err) { + log.G(ctx).Debugf("integrity digest for blob already exists") + return err + } + } + + digestFile, err := os.Create(digestPath) + if err != nil { + if os.IsExist(err) { + log.G(ctx).Debugf("Error creating integrity digest file: digest for blob already exists") + return err + } + + return fmt.Errorf("Error creating integrity digest file for blob: %s", dgst.Encoded()) + } + + _, err = digestFile.WriteString(verityDigest) + if err != nil { + log.G(ctx).Debugf("Error writing fsverity digest to file: %s", err) + return err + } + + return nil +} From 4c807107509ab6bc76ce0db723bdc58878ce4b9c Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Tue, 13 Feb 2024 16:27:01 -0500 Subject: [PATCH 32/40] Remove old logging messages Remove old logs for debugging fsverity. Cannot call the log function any more because logging depends on having the context. The error returned by the integrity store and integrity measure functions are logged instead. --- content/local/store.go | 8 +------- content/local/writer.go | 15 ++++----------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index ae60a4dc9488..2794e7492ae8 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -706,10 +706,7 @@ func storeIntegrity(rootPath string, p string, desc ocispec.Descriptor) error { return fmt.Errorf("integrity validation is not supported") } - log.G(ctx).Debugf("verifying blob with fsverity") - var verityDigest string - log.G(ctx).Debugf("measuring blob: %s", p) verityDigest, merr := fsverity.Measure(p) if merr != nil { return fmt.Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) @@ -719,20 +716,17 @@ func storeIntegrity(rootPath string, p string, desc ocispec.Descriptor) error { integrityFile := filepath.Join(rootPath, "integrity", desc.Digest.Encoded()) ifd, err := os.Open(integrityFile) if err != nil { - log.G(ctx).Errorf("failed to read integrity file of blob %s", p) return fmt.Errorf("could not read expected integrity value of %s", p) } b, err := io.ReadAll(ifd) if err != nil { - log.G(ctx).Errorf("could not read fsverity digest from integrity file: %s", err.Error()) + return fmt.Errorf("could not read fsverity digest from integrity file: %s", err.Error()) } else { expectedDigest = string(b) } // compare the digest to the "good" value stored in the blob label - log.G(ctx).Debugf("comparing measured digest to known good value") if verityDigest != expectedDigest { - log.G(ctx).Errorf("Error: fsverity digest does not match the expected digest") return fmt.Errorf("blob not trusted: fsverity digest does not match the expected digest value") } diff --git a/content/local/writer.go b/content/local/writer.go index 9ed2901c5d56..4fdfaa25c75d 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -219,7 +219,6 @@ func validateIntegrity(rootPath string, target string, dgst digest.Digest) error } var verityDigest string - log.G(ctx).Debugf("enabling fsverity on blob %v", target) if err := fsverity.Enable(target); err != nil { return fmt.Errorf("failed to enable fsverity verification: %s", err.Error()) } @@ -229,28 +228,23 @@ func validateIntegrity(rootPath string, target string, dgst digest.Digest) error return fmt.Errorf("failed to take fsverity measurement of blob: %s", merr.Error()) } - log.G(ctx).Debugf("storing \"good\" digest value") - integrityStore := filepath.Join(rootPath, "integrity") if err := os.MkdirAll(integrityStore, 0755); err != nil { - log.G(ctx).Debugf("error creating integrity digest directory: %s", err.Error()) - return err + return fmt.Errorf("error creating integrity digest directory: %s", err) } digestPath := filepath.Join(integrityStore, dgst.Encoded()) _, err := os.Stat(digestPath) if err != nil { if os.IsExist(err) { - log.G(ctx).Debugf("integrity digest for blob already exists") - return err + return fmt.Errorf("integrity digest for blob already exists: %s", err) } } digestFile, err := os.Create(digestPath) if err != nil { if os.IsExist(err) { - log.G(ctx).Debugf("Error creating integrity digest file: digest for blob already exists") - return err + return fmt.Errorf("error createing integrity digest file: %s", err) } return fmt.Errorf("Error creating integrity digest file for blob: %s", dgst.Encoded()) @@ -258,8 +252,7 @@ func validateIntegrity(rootPath string, target string, dgst digest.Digest) error _, err = digestFile.WriteString(verityDigest) if err != nil { - log.G(ctx).Debugf("Error writing fsverity digest to file: %s", err) - return err + return fmt.Errorf("error writing vsverity digest to file: %s", err) } return nil From d1e074c5ec1b38b1b5d4fde52c538543b1f8e2fd Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Wed, 14 Feb 2024 15:20:59 -0500 Subject: [PATCH 33/40] Make fsverity IsSupported function more robust Check if fsverity is able to be executed on a file instead of assuming that all linux systems have fsverity capabilities. Change names of store integrity and validate integrity functions to be accurate. --- content/local/store.go | 8 ++++---- content/local/writer.go | 10 +++++----- pkg/fsverity/fsverity_linux.go | 25 +++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index 2794e7492ae8..84cfd90fb5fb 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -130,7 +130,7 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. } log.G(ctx).Debugf("Getting reader for blob %v", p) - if err = storeIntegrity(s.root, p, desc); err != nil { + if err = validateIntegrity(s.root, p, desc); err != nil { log.G(ctx).Errorf("error storing integrity value of blob %v: %s", p, err.Error()) } @@ -700,10 +700,10 @@ func writeToCompletion(path string, data []byte, mode os.FileMode) error { return nil } -func storeIntegrity(rootPath string, p string, desc ocispec.Descriptor) error { +func validateIntegrity(rootPath string, p string, desc ocispec.Descriptor) error { // validate the integrity of the blob if integrity validation is supported - if !fsverity.IsSupported() { - return fmt.Errorf("integrity validation is not supported") + if supported, err := fsverity.IsSupported(rootPath); !supported { + return fmt.Errorf("integrity validation is not supported: %s", err.Error()) } var verityDigest string diff --git a/content/local/writer.go b/content/local/writer.go index 4fdfaa25c75d..8e927145d99c 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -138,7 +138,7 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, return err } - if err = validateIntegrity(w.s.root, target, dgst); err != nil { + if err = storeIntegrity(w.s.root, target, dgst); err != nil { log.G(ctx).Errorf("failed to validate integrity of blob %v: %s", target, err.Error()) } @@ -213,9 +213,9 @@ func (w *writer) Truncate(size int64) error { return w.fp.Truncate(0) } -func validateIntegrity(rootPath string, target string, dgst digest.Digest) error { - if !fsverity.IsSupported() { - return fmt.Errorf("integrity validation is not supported") +func storeIntegrity(rootPath string, target string, dgst digest.Digest) error { + if supported, err := fsverity.IsSupported(rootPath); !supported { + return fmt.Errorf("integrity validation is not supported: %s", err.Error()) } var verityDigest string @@ -244,7 +244,7 @@ func validateIntegrity(rootPath string, target string, dgst digest.Digest) error digestFile, err := os.Create(digestPath) if err != nil { if os.IsExist(err) { - return fmt.Errorf("error createing integrity digest file: %s", err) + return fmt.Errorf("error creating integrity digest file: %s", err) } return fmt.Errorf("Error creating integrity digest file for blob: %s", dgst.Encoded()) diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index e7a8b1850504..61aa0128733a 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -21,6 +21,7 @@ package fsverity import ( "fmt" "os" + "path/filepath" "syscall" "unsafe" @@ -50,8 +51,28 @@ const ( maxDigestSize uint16 = 64 ) -func IsSupported() bool { - return true +func IsSupported(rootPath string) (bool, error) { + integrityStore := filepath.Join(rootPath, "integrity") + if err := os.MkdirAll(integrityStore, 0755); err != nil { + return false, fmt.Errorf("error creating integrity digest directory: %s", err.Error()) + } + + digestPath := filepath.Join(integrityStore, "supported") + _, err := os.Create(digestPath) + if err != nil { + if os.IsExist(err) { + return false, fmt.Errorf("error creating integrity digest file: %s", err) + } + return false, fmt.Errorf("error creating integrity digest file") + } + defer os.Remove(digestPath) + + eerr := Enable(digestPath) + if eerr != nil { + return false, fmt.Errorf("fsverity not supported: %s", eerr.Error()) + } + + return true, nil } func IsEnabled(path string) (bool, error) { From 1017208976d6e59e2f7ddb9234303869e568749e Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Wed, 14 Feb 2024 16:39:05 -0500 Subject: [PATCH 34/40] Close test verity file before enabling fsverity Enabling fsverity fails because test file is open and in use. Close the file before calling fsverity operations. --- content/local/store.go | 2 +- content/local/writer.go | 2 +- pkg/fsverity/fsverity_linux.go | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index 84cfd90fb5fb..a7bf11a2c944 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -131,7 +131,7 @@ func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content. log.G(ctx).Debugf("Getting reader for blob %v", p) if err = validateIntegrity(s.root, p, desc); err != nil { - log.G(ctx).Errorf("error storing integrity value of blob %v: %s", p, err.Error()) + log.G(ctx).Errorf("error validating integrity value of blob %v: %s", p, err.Error()) } reader, err := OpenReader(p) diff --git a/content/local/writer.go b/content/local/writer.go index 8e927145d99c..3bf28ffc93ec 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -139,7 +139,7 @@ func (w *writer) Commit(ctx context.Context, size int64, expected digest.Digest, } if err = storeIntegrity(w.s.root, target, dgst); err != nil { - log.G(ctx).Errorf("failed to validate integrity of blob %v: %s", target, err.Error()) + log.G(ctx).Errorf("failed to store integrity of blob %v: %s", target, err.Error()) } // Ingest has now been made available in the content store, attempt to complete diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index 61aa0128733a..8251784d2f24 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -58,13 +58,14 @@ func IsSupported(rootPath string) (bool, error) { } digestPath := filepath.Join(integrityStore, "supported") - _, err := os.Create(digestPath) + digestFile, err := os.Create(digestPath) if err != nil { if os.IsExist(err) { return false, fmt.Errorf("error creating integrity digest file: %s", err) } return false, fmt.Errorf("error creating integrity digest file") } + digestFile.Close() defer os.Remove(digestPath) eerr := Enable(digestPath) From f6b8e5a79d7816ae5748486ae3de58040eb3226e Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Wed, 14 Feb 2024 16:54:12 -0500 Subject: [PATCH 35/40] Modify IsSupported definition Modify the definition of IsSupported fsverity function for non-linux platforms to match the definition of the linux IsSupported function. --- pkg/fsverity/fsverity_linux.go | 1 + pkg/fsverity/fsverity_other.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index 8251784d2f24..d50a033bff6a 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -51,6 +51,7 @@ const ( maxDigestSize uint16 = 64 ) +// IsSupported ensures that fsverity is enabled on the filesystem where the content blobs are stored. func IsSupported(rootPath string) (bool, error) { integrityStore := filepath.Join(rootPath, "integrity") if err := os.MkdirAll(integrityStore, 0755); err != nil { diff --git a/pkg/fsverity/fsverity_other.go b/pkg/fsverity/fsverity_other.go index c6c2e1a47b9d..5e27ddaa7c1d 100644 --- a/pkg/fsverity/fsverity_other.go +++ b/pkg/fsverity/fsverity_other.go @@ -2,6 +2,6 @@ package fsverity -func IsSupported() bool { - return false +func IsSupported(_ string) (bool, error) { + return false, nil } From 5b52535aae77c43f7f1bdb1d61c0b4619c0e3145 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Thu, 15 Feb 2024 10:29:02 -0500 Subject: [PATCH 36/40] Modify fsverity IsSupported Modify IsSupported function to determine support based on the Linux kernel version. --- content/local/store.go | 4 ++-- content/local/writer.go | 4 ++-- pkg/fsverity/fsverity_linux.go | 30 +++++++----------------------- pkg/fsverity/fsverity_other.go | 4 ++-- 4 files changed, 13 insertions(+), 29 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index a7bf11a2c944..7f5a9b86c050 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -702,8 +702,8 @@ func writeToCompletion(path string, data []byte, mode os.FileMode) error { func validateIntegrity(rootPath string, p string, desc ocispec.Descriptor) error { // validate the integrity of the blob if integrity validation is supported - if supported, err := fsverity.IsSupported(rootPath); !supported { - return fmt.Errorf("integrity validation is not supported: %s", err.Error()) + if supported := fsverity.IsSupported(); !supported { + return fmt.Errorf("integrity validation is not supported") } var verityDigest string diff --git a/content/local/writer.go b/content/local/writer.go index 3bf28ffc93ec..def23834483e 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -214,8 +214,8 @@ func (w *writer) Truncate(size int64) error { } func storeIntegrity(rootPath string, target string, dgst digest.Digest) error { - if supported, err := fsverity.IsSupported(rootPath); !supported { - return fmt.Errorf("integrity validation is not supported: %s", err.Error()) + if supported := fsverity.IsSupported(); !supported { + return fmt.Errorf("integrity validation is not supported") } var verityDigest string diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index d50a033bff6a..4531ed08fe6c 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -20,8 +20,8 @@ package fsverity import ( "fmt" + "github.com/containerd/containerd/v2/contrib/seccomp/kernelversion" "os" - "path/filepath" "syscall" "unsafe" @@ -51,30 +51,14 @@ const ( maxDigestSize uint16 = 64 ) -// IsSupported ensures that fsverity is enabled on the filesystem where the content blobs are stored. -func IsSupported(rootPath string) (bool, error) { - integrityStore := filepath.Join(rootPath, "integrity") - if err := os.MkdirAll(integrityStore, 0755); err != nil { - return false, fmt.Errorf("error creating integrity digest directory: %s", err.Error()) - } - - digestPath := filepath.Join(integrityStore, "supported") - digestFile, err := os.Create(digestPath) +// TODO: Use once to set a constant instead??? +func IsSupported() bool { + minKernelVersion := kernelversion.KernelVersion{Kernel: 5, Major: 4} + supported, err := kernelversion.GreaterEqualThan(minKernelVersion) if err != nil { - if os.IsExist(err) { - return false, fmt.Errorf("error creating integrity digest file: %s", err) - } - return false, fmt.Errorf("error creating integrity digest file") + return false } - digestFile.Close() - defer os.Remove(digestPath) - - eerr := Enable(digestPath) - if eerr != nil { - return false, fmt.Errorf("fsverity not supported: %s", eerr.Error()) - } - - return true, nil + return supported } func IsEnabled(path string) (bool, error) { diff --git a/pkg/fsverity/fsverity_other.go b/pkg/fsverity/fsverity_other.go index 5e27ddaa7c1d..c6c2e1a47b9d 100644 --- a/pkg/fsverity/fsverity_other.go +++ b/pkg/fsverity/fsverity_other.go @@ -2,6 +2,6 @@ package fsverity -func IsSupported(_ string) (bool, error) { - return false, nil +func IsSupported() bool { + return false } From 6f6a3e6cbdab9e0934ad41727e7e2f5928bac59a Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Thu, 15 Feb 2024 10:57:05 -0500 Subject: [PATCH 37/40] Use sync once with fsverity IsSupported Use sync Once to check the kernel verision a single time, avoiding redundant work. --- pkg/fsverity/fsverity_linux.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index 4531ed08fe6c..00a767ccfe89 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -20,11 +20,12 @@ package fsverity import ( "fmt" - "github.com/containerd/containerd/v2/contrib/seccomp/kernelversion" "os" + "sync" "syscall" "unsafe" + "github.com/containerd/containerd/v2/contrib/seccomp/kernelversion" "golang.org/x/sys/unix" ) @@ -51,13 +52,20 @@ const ( maxDigestSize uint16 = 64 ) -// TODO: Use once to set a constant instead??? +var ( + once sync.Once + supported bool +) + func IsSupported() bool { - minKernelVersion := kernelversion.KernelVersion{Kernel: 5, Major: 4} - supported, err := kernelversion.GreaterEqualThan(minKernelVersion) - if err != nil { - return false - } + once.Do(func () { + minKernelVersion := kernelversion.KernelVersion{Kernel: 5, Major: 4} + s, err := kernelversion.GreaterEqualThan(minKernelVersion) + if err != nil { + supported = false + } + supported = s + }) return supported } From 5644a40bdad65df25cd8ba217a62a6b36a6625c0 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Fri, 16 Feb 2024 09:27:07 -0500 Subject: [PATCH 38/40] Modify fsverity struct field names Change field names from snake case to camel case. --- pkg/fsverity/fsverity_linux.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index 00a767ccfe89..82dae8fd199b 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -31,19 +31,19 @@ import ( type fsverityEnableArg struct { version uint32 - hash_algorithm uint32 - block_size uint32 - salt_size uint32 - salt_ptr uint64 - sig_size uint32 + hashAlgorithm uint32 + blockSize uint32 + saltSize uint32 + saltPtr uint64 + sigSize uint32 reserved1 uint32 - sig_ptr uint64 + sigPtr uint64 reserved2 [11]uint64 } type fsverityDigest struct { - digest_algorithm uint16 - digest_size uint16 + digestAlgorithm uint16 + digestSize uint16 digest [64]uint8 } @@ -97,7 +97,7 @@ func Enable(path string) error { var args *fsverityEnableArg = &fsverityEnableArg{} args.version = 1 - args.hash_algorithm = 1 + args.hashAlgorithm = 1 // fsverity block size should be the minimum between the page size // and the file system block size @@ -114,7 +114,7 @@ func Enable(path string) error { blockSize = defaultBlockSize } - args.block_size = uint32(blockSize) + args.blockSize = uint32(blockSize) _, _, errno := unix.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(unix.FS_IOC_ENABLE_VERITY), uintptr(unsafe.Pointer(args))) if errno != 0 { @@ -131,14 +131,14 @@ func Measure(path string) (string, error) { return verityDigest, fmt.Errorf("Error opening file: %s\n", err.Error()) } - var d *fsverityDigest = &fsverityDigest{digest_size: maxDigestSize} + var d *fsverityDigest = &fsverityDigest{digestSize: maxDigestSize} _, _, errno := unix.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(unix.FS_IOC_MEASURE_VERITY), uintptr(unsafe.Pointer(d))) if errno != 0 { return verityDigest, fmt.Errorf("Measure fsverity failed: %d\n", errno) } var i uint16 - for i = 0; i < (*d).digest_size; i++ { + for i = 0; i < (*d).digestSize; i++ { verityDigest = fmt.Sprintf("%s%x", verityDigest, (*d).digest[i]) } From 23a1e3dffa921c495ff64be73c5afd031355bdee Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Fri, 16 Feb 2024 12:05:37 -0500 Subject: [PATCH 39/40] Check if fsverity is enabled on the filesystem Use fsverity enable function to check if fsverity is active on the filesystem where the content blobs are stored. --- pkg/fsverity/fsverity_linux.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index 82dae8fd199b..e7e2b706beb0 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -21,6 +21,7 @@ package fsverity import ( "fmt" "os" + "path/filepath" "sync" "syscall" "unsafe" @@ -57,14 +58,36 @@ var ( supported bool ) -func IsSupported() bool { +func IsSupported(rootPath string) bool { once.Do(func () { minKernelVersion := kernelversion.KernelVersion{Kernel: 5, Major: 4} s, err := kernelversion.GreaterEqualThan(minKernelVersion) + if err != nil { + supported = s + return + } + + integrityStore := filepath.Join(rootPath, "integrity") + if err = os.MkdirAll(integrityStore, 0755); err != nil { + supported = false + return + } + + digestPath := filepath.Join(integrityStore, "supported") + _, err = os.Create(digestPath) if err != nil { supported = false + return } - supported = s + defer os.Remove(digestPath) + + eerr := Enable(digestPath) + if eerr != nil { + supported = false + return + } + + supported = true }) return supported } From 3c37f6ec9b022acba200a1dff6ff07a24e679410 Mon Sep 17 00:00:00 2001 From: James Jenkins Date: Fri, 16 Feb 2024 12:05:37 -0500 Subject: [PATCH 40/40] Check if fsverity is enabled on the filesystem Use fsverity enable function to check if fsverity is active on the filesystem where the content blobs are stored. --- content/local/store.go | 2 +- content/local/writer.go | 2 +- pkg/fsverity/fsverity_linux.go | 28 ++++++++++++++++++++++++++-- pkg/fsverity/fsverity_other.go | 2 +- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/content/local/store.go b/content/local/store.go index 7f5a9b86c050..58cc295aa034 100644 --- a/content/local/store.go +++ b/content/local/store.go @@ -702,7 +702,7 @@ func writeToCompletion(path string, data []byte, mode os.FileMode) error { func validateIntegrity(rootPath string, p string, desc ocispec.Descriptor) error { // validate the integrity of the blob if integrity validation is supported - if supported := fsverity.IsSupported(); !supported { + if supported := fsverity.IsSupported(rootPath); !supported { return fmt.Errorf("integrity validation is not supported") } diff --git a/content/local/writer.go b/content/local/writer.go index def23834483e..e95989ef3eb4 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -214,7 +214,7 @@ func (w *writer) Truncate(size int64) error { } func storeIntegrity(rootPath string, target string, dgst digest.Digest) error { - if supported := fsverity.IsSupported(); !supported { + if supported := fsverity.IsSupported(rootPath); !supported { return fmt.Errorf("integrity validation is not supported") } diff --git a/pkg/fsverity/fsverity_linux.go b/pkg/fsverity/fsverity_linux.go index 82dae8fd199b..9c01c5cc4ed0 100644 --- a/pkg/fsverity/fsverity_linux.go +++ b/pkg/fsverity/fsverity_linux.go @@ -21,6 +21,7 @@ package fsverity import ( "fmt" "os" + "path/filepath" "sync" "syscall" "unsafe" @@ -57,14 +58,37 @@ var ( supported bool ) -func IsSupported() bool { +func IsSupported(rootPath string) bool { once.Do(func () { minKernelVersion := kernelversion.KernelVersion{Kernel: 5, Major: 4} s, err := kernelversion.GreaterEqualThan(minKernelVersion) + if err != nil { + supported = s + return + } + + integrityStore := filepath.Join(rootPath, "integrity") + if err = os.MkdirAll(integrityStore, 0755); err != nil { + supported = false + return + } + + digestPath := filepath.Join(integrityStore, "supported") + digestFile, err := os.Create(digestPath) if err != nil { supported = false + return } - supported = s + digestFile.Close() + defer os.Remove(digestPath) + + eerr := Enable(digestPath) + if eerr != nil { + supported = false + return + } + + supported = true }) return supported } diff --git a/pkg/fsverity/fsverity_other.go b/pkg/fsverity/fsverity_other.go index c6c2e1a47b9d..eafd144232f8 100644 --- a/pkg/fsverity/fsverity_other.go +++ b/pkg/fsverity/fsverity_other.go @@ -2,6 +2,6 @@ package fsverity -func IsSupported() bool { +func IsSupported(rootPath string) bool { return false }