Feat/download foreign layers

cmd/skopeo/utils.go

@@ -338,6 +338,7 @@ type sharedCopyOptions struct {
 	signBySigstorePrivateKey string                    // Sign the image using a sigstore private key
 	signPassphraseFile       string                    // Path pointing to a passphrase file when signing
 	preserveDigests          bool                      // Preserve digests during copy
+	downloadForeignLayers    bool                      // Download foreign layers during copy
 	format                   commonFlag.OptionalString // Force conversion of the image to a specified format
 }
 
@@ -353,6 +354,7 @@ func sharedCopyFlags() (pflag.FlagSet, *sharedCopyOptions) {
 	fs.StringVar(&opts.signPassphraseFile, "sign-passphrase-file", "", "Read a passphrase for signing an image from `PATH`")
 	fs.VarP(commonFlag.NewOptionalStringValue(&opts.format), "format", "f", `MANIFEST TYPE (oci, v2s1, or v2s2) to use in the destination (default is manifest type of source, with fallbacks)`)
 	fs.BoolVar(&opts.preserveDigests, "preserve-digests", false, "Preserve digests of images and lists")
+	fs.BoolVar(&opts.downloadForeignLayers, "download-foreign-layers", false, "Download nondistributable (foreign) layers")
 	return fs, &opts
 }
 
@@ -462,6 +464,7 @@ func (opts *sharedCopyOptions) copyOptions(stdout io.Writer) (*copy.Options, fun
 		ReportWriter: stdout,
 
 		PreserveDigests:       opts.preserveDigests,
+		DownloadForeignLayers: opts.downloadForeignLayers,
 		ForceManifestMIMEType: manifestType,
 	}, closeSigners, nil
 }

cmd/skopeo/utils_test.go

@@ -395,12 +395,14 @@ func TestSharedCopyOptionsCopyOptions(t *testing.T) {
 				"--sign-by", "gpgFingerprint",
 				"--format", "oci",
 				"--preserve-digests",
+				"--download-foreign-layers",
 			},
 			expected: copy.Options{
 				RemoveSignatures:      true,
 				SignBy:                "gpgFingerprint",
 				ReportWriter:          &someStdout,
 				PreserveDigests:       true,
+				DownloadForeignLayers: true,
 				ForceManifestMIMEType: imgspecv1.MediaTypeImageManifest,
 			},
 		},

docs/skopeo-copy.1.md

@@ -63,6 +63,12 @@ Preserve the digests during copying. Fail if the digest cannot be preserved.
 
 This option does not change what will be copied; consider using `--all` at the same time.
 
+**--download-foreign-layers**
+
+Download layer contents for nondistributable ("foreign") layers — e.g. Windows base layers — that would otherwise be skipped in favor of URL references to their original location. By default, foreign layers are not downloaded.
+
+Useful for air-gapped or offline copies of images whose original source may later become unreachable. Note that the destination manifest is not rewritten: layers downloaded by this option still appear with their original nondistributable media type and `urls` field, so consumers that strictly reject manifests containing foreign URLs are not helped by this option alone.
+
 **--encrypt-layer** _ints_
 
 *Experimental* the 0-indexed layer indices, with support for negative indexing (e.g. 0 is the first layer, -1 is the last layer)

docs/skopeo-sync.1.md

@@ -88,6 +88,12 @@ Preserve the digests during copying. Fail if the digest cannot be preserved.
 
 This option does not change what will be copied; consider using `--all` at the same time.
 
+**--download-foreign-layers**
+
+Download layer contents for nondistributable ("foreign") layers — e.g. Windows base layers — that would otherwise be skipped in favor of URL references to their original location. By default, foreign layers are not downloaded.
+
+Useful for air-gapped or offline copies of images whose original source may later become unreachable. The destination manifest is not rewritten: layers retain their original nondistributable media type and `urls` field.
+
 **--remove-signatures** Do not copy signatures, if any, from _source-image_. This is necessary when copying a signed image to a destination which does not support signatures.
 
 **--sign-by** _key-id_

integration/copy_test.go

@@ -503,6 +503,85 @@ func (s *copySuite) TestCopySucceedsWhenImageDoesNotMatchRuntimeButWeOverride()
 		"containers-storage:"+storage+"test")
 }
 
+// TestCopyDownloadForeignLayers verifies that --download-foreign-layers downloads
+// the contents of nondistributable ("foreign") layers into the destination, and
+// that without the flag those layers are skipped. The destination is an oci:
+// layout, which is the case the original feature request (containers/skopeo#545)
+// was filed for: oci: accepts foreign-URL references, so without the flag the
+// library deliberately preserves them in the manifest and skips downloading the
+// blob.
+//
+// The library's doc-comment for copy.Options.DownloadForeignLayers also claims to
+// "translate the layer media type to not indicate 'nondistributable'", but
+// empirically this manifest rewrite does not happen for oci: destinations as of
+// the vendored containers/image at the time of this commit: the manifest is
+// byte-identical before and after the flag. This test therefore asserts only the
+// behavior we can observe (blob present vs. absent), leaving manifest contents
+// out of scope. If the upstream library later closes that gap, this test will
+// continue to pass.
+func (s *copySuite) TestCopyDownloadForeignLayers() {
+	t := s.T()
+	withoutFlag := t.TempDir()
+	withFlag := t.TempDir()
+
+	// Default behavior: oci: accepts foreign URLs, so the manifest preserves them
+	// and the foreign blob is not downloaded.
+	assertSkopeoSucceeds(t, "", "--override-os=windows", "--override-arch=amd64",
+		"copy", "--retry-times", "3", knownWindowsOnlyImage, "oci:"+withoutFlag)
+	assertOCILayoutForeignBlobs(t, withoutFlag, false)
+
+	// With --download-foreign-layers: the foreign blob is fetched into the layout.
+	assertSkopeoSucceeds(t, "", "--override-os=windows", "--override-arch=amd64",
+		"copy", "--retry-times", "3", "--download-foreign-layers", knownWindowsOnlyImage,
+		"oci:"+withFlag)
+	assertOCILayoutForeignBlobs(t, withFlag, true)
+}
+
+// assertOCILayoutForeignBlobs inspects the oci: layout at dir, identifies the
+// foreign layers in its (single) image manifest by the presence of a non-empty
+// urls array, and asserts whether each foreign layer's blob file is present on
+// disk. This is the user-visible effect of --download-foreign-layers.
+func assertOCILayoutForeignBlobs(t *testing.T, dir string, expectPresent bool) {
+	indexBlob, err := os.ReadFile(filepath.Join(dir, "index.json"))
+	require.NoError(t, err)
+	var index struct {
+		Manifests []struct {
+			Digest digest.Digest `json:"digest"`
+		} `json:"manifests"`
+	}
+	require.NoError(t, json.Unmarshal(indexBlob, &index))
+	require.Len(t, index.Manifests, 1, "expected exactly one manifest in oci: index; index: %s", indexBlob)
+
+	manifestDigest := index.Manifests[0].Digest
+	manifestBlob, err := os.ReadFile(filepath.Join(dir, "blobs", manifestDigest.Algorithm().String(), manifestDigest.Encoded()))
+	require.NoError(t, err)
+	var m struct {
+		Layers []struct {
+			Digest digest.Digest `json:"digest"`
+			URLs   []string      `json:"urls,omitempty"`
+		} `json:"layers"`
+	}
+	require.NoError(t, json.Unmarshal(manifestBlob, &m))
+
+	var foreignDigests []digest.Digest
+	for _, l := range m.Layers {
+		if len(l.URLs) > 0 {
+			foreignDigests = append(foreignDigests, l.Digest)
+		}
+	}
+	require.NotEmpty(t, foreignDigests, "test image was expected to have at least one foreign layer; manifest: %s", manifestBlob)
+
+	for _, d := range foreignDigests {
+		blobPath := filepath.Join(dir, "blobs", d.Algorithm().String(), d.Encoded())
+		_, statErr := os.Stat(blobPath)
+		if expectPresent {
+			assert.NoError(t, statErr, "expected foreign-layer blob %s to be present at %s", d, blobPath)
+		} else {
+			assert.True(t, os.IsNotExist(statErr), "expected foreign-layer blob %s to be absent at %s; stat err: %v", d, blobPath, statErr)
+		}
+	}
+}
+
 func (s *copySuite) TestCopySimpleAtomicRegistry() {
 	t := s.T()
 	dir1 := t.TempDir()