diff --git a/container.go b/container.go index d8668740f6..cd8de854b2 100644 --- a/container.go +++ b/container.go @@ -19,9 +19,9 @@ import ( "github.com/docker/docker/pkg/archive" "github.com/docker/go-connections/nat" "github.com/google/uuid" - "github.com/moby/patternmatcher/ignorefile" tcexec "github.com/testcontainers/testcontainers-go/exec" + "github.com/testcontainers/testcontainers-go/image" "github.com/testcontainers/testcontainers-go/internal/core" "github.com/testcontainers/testcontainers-go/log" "github.com/testcontainers/testcontainers-go/wait" @@ -71,18 +71,9 @@ type Container interface { GetLogProductionErrorChannel() <-chan error } +// Deprecated: Use image.Builder instead // ImageBuildInfo defines what is needed to build an image -type ImageBuildInfo interface { - BuildOptions() (types.ImageBuildOptions, error) // converts the ImageBuildInfo to a types.ImageBuildOptions - GetContext() (io.Reader, error) // the path to the build context - GetDockerfile() string // the relative path to the Dockerfile, including the file itself - GetRepo() string // get repo label for image - GetTag() string // get tag label for image - BuildLogWriter() io.Writer // for output of build log, use io.Discard to disable the output - ShouldBuildImage() bool // return true if the image needs to be built - GetBuildArgs() map[string]*string // return the environment args used to build the Dockerfile - GetAuthConfigs() map[string]registry.AuthConfig // Deprecated. Testcontainers will detect registry credentials automatically. Return the auth configs to be able to pull from an authenticated docker registry -} +type ImageBuildInfo = image.Builder // FromDockerfile represents the parameters needed to build an image from a Dockerfile // rather than using a pre-built one @@ -131,7 +122,7 @@ type ContainerRequest struct { FromDockerfile HostAccessPorts []int Image string - ImageSubstitutors []ImageSubstitutor + ImageSubstitutors []image.Substitutor Entrypoint []string Env map[string]string ExposedPorts []string // allow specifying protocol info @@ -240,7 +231,7 @@ func (c *ContainerRequest) GetContext() (io.Reader, error) { } c.Context = abs - dockerIgnoreExists, excluded, err := parseDockerIgnore(abs) + dockerIgnoreExists, excluded, err := image.ParseDockerIgnore(abs) if err != nil { return nil, err } @@ -263,26 +254,6 @@ func (c *ContainerRequest) GetContext() (io.Reader, error) { return buildContext, nil } -// parseDockerIgnore returns if the file exists, the excluded files and an error if any -func parseDockerIgnore(targetDir string) (bool, []string, error) { - // based on https://github.com/docker/cli/blob/master/cli/command/image/build/dockerignore.go#L14 - fileLocation := filepath.Join(targetDir, ".dockerignore") - var excluded []string - exists := false - if f, openErr := os.Open(fileLocation); openErr == nil { - defer f.Close() - - exists = true - - var err error - excluded, err = ignorefile.ReadAll(f) - if err != nil { - return true, excluded, fmt.Errorf("error reading .dockerignore: %w", err) - } - } - return exists, excluded, nil -} - // GetBuildArgs returns the env args to be used when creating from Dockerfile func (c *ContainerRequest) GetBuildArgs() map[string]*string { return c.FromDockerfile.BuildArgs diff --git a/container_ignore_test.go b/container_ignore_test.go deleted file mode 100644 index 505b9edd6d..0000000000 --- a/container_ignore_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// This test is testing very internal logic that should not be exported away from this package. We'll -// leave it in the main testcontainers package. Do not use for user facing examples. -package testcontainers - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestParseDockerIgnore(t *testing.T) { - testCases := []struct { - filePath string - exists bool - expectedErr error - expectedExcluded []string - }{ - { - filePath: "./testdata/dockerignore", - expectedErr: nil, - exists: true, - expectedExcluded: []string{"vendor", "foo", "bar"}, - }, - { - filePath: "./testdata", - expectedErr: nil, - exists: true, - expectedExcluded: []string{"Dockerfile", "echo.Dockerfile"}, - }, - { - filePath: "./testdata/data", - expectedErr: nil, - expectedExcluded: nil, // it's nil because the parseDockerIgnore function uses the zero value of a slice - }, - } - - for _, testCase := range testCases { - exists, excluded, err := parseDockerIgnore(testCase.filePath) - assert.Equal(t, testCase.exists, exists) - require.ErrorIs(t, testCase.expectedErr, err) - assert.Equal(t, testCase.expectedExcluded, excluded) - } -} diff --git a/container_test.go b/container_test.go index 5a6a68ff5b..e886af3f6b 100644 --- a/container_test.go +++ b/container_test.go @@ -4,7 +4,6 @@ import ( "archive/tar" "bytes" "context" - "errors" "fmt" "io" "testing" @@ -16,6 +15,7 @@ import ( "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/image" "github.com/testcontainers/testcontainers-go/log" "github.com/testcontainers/testcontainers-go/wait" ) @@ -400,112 +400,6 @@ func Test_GetLogsFromFailedContainer(t *testing.T) { require.Contains(t, log, "I was not expecting this") } -// dockerImageSubstitutor { -type dockerImageSubstitutor struct{} - -func (s dockerImageSubstitutor) Description() string { - return "DockerImageSubstitutor (prepends registry.hub.docker.com)" -} - -func (s dockerImageSubstitutor) Substitute(image string) (string, error) { - return "registry.hub.docker.com/library/" + image, nil -} - -// } - -// noopImageSubstitutor { -type NoopImageSubstitutor struct{} - -// Description returns a description of what is expected from this Substitutor, -// which is used in logs. -func (s NoopImageSubstitutor) Description() string { - return "NoopImageSubstitutor (noop)" -} - -// Substitute returns the original image, without any change -func (s NoopImageSubstitutor) Substitute(image string) (string, error) { - return image, nil -} - -// } - -type errorSubstitutor struct{} - -var errSubstitution = errors.New("substitution error") - -// Description returns a description of what is expected from this Substitutor, -// which is used in logs. -func (s errorSubstitutor) Description() string { - return "errorSubstitutor" -} - -// Substitute returns the original image, but returns an error -func (s errorSubstitutor) Substitute(image string) (string, error) { - return image, errSubstitution -} - -func TestImageSubstitutors(t *testing.T) { - tests := []struct { - name string - image string // must be a valid image, as the test will try to create a container from it - substitutors []testcontainers.ImageSubstitutor - expectedImage string - expectedError error - }{ - { - name: "No substitutors", - image: "alpine", - expectedImage: "alpine", - }, - { - name: "Noop substitutor", - image: "alpine", - substitutors: []testcontainers.ImageSubstitutor{NoopImageSubstitutor{}}, - expectedImage: "alpine", - }, - { - name: "Prepend namespace", - image: "alpine", - substitutors: []testcontainers.ImageSubstitutor{dockerImageSubstitutor{}}, - expectedImage: "registry.hub.docker.com/library/alpine", - }, - { - name: "Substitution with error", - image: "alpine", - substitutors: []testcontainers.ImageSubstitutor{errorSubstitutor{}}, - expectedImage: "alpine", - expectedError: errSubstitution, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ctx := context.Background() - req := testcontainers.ContainerRequest{ - Image: test.image, - ImageSubstitutors: test.substitutors, - } - - ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - testcontainers.CleanupContainer(t, ctr) - if test.expectedError != nil { - require.ErrorIs(t, err, test.expectedError) - return - } - - require.NoError(t, err) - - // enforce the concrete type, as GenericContainer returns an interface, - // which will be changed in future implementations of the library - dockerContainer := ctr.(*testcontainers.DockerContainer) - assert.Equal(t, test.expectedImage, dockerContainer.Image) - }) - } -} - func TestShouldStartContainersInParallel(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) t.Cleanup(cancel) @@ -544,7 +438,7 @@ func ExampleGenericContainer_withSubstitutors() { ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: testcontainers.ContainerRequest{ Image: "alpine:latest", - ImageSubstitutors: []testcontainers.ImageSubstitutor{dockerImageSubstitutor{}}, + ImageSubstitutors: []image.Substitutor{image.DockerSubstitutor{}}, }, Started: true, }) diff --git a/docker.go b/docker.go index 4e8bf38d7c..bb51680f10 100644 --- a/docker.go +++ b/docker.go @@ -27,13 +27,12 @@ import ( "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/docker/docker/errdefs" - "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/stdcopy" "github.com/docker/go-connections/nat" - "github.com/moby/term" specs "github.com/opencontainers/image-spec/specs-go/v1" tcexec "github.com/testcontainers/testcontainers-go/exec" + tcimage "github.com/testcontainers/testcontainers-go/image" "github.com/testcontainers/testcontainers-go/internal/config" "github.com/testcontainers/testcontainers-go/internal/core" "github.com/testcontainers/testcontainers-go/log" @@ -975,51 +974,10 @@ func (p *DockerProvider) SetClient(c client.APIClient) { var _ ContainerProvider = (*DockerProvider)(nil) +// Deprecated: use image.Build instead // BuildImage will build and image from context and Dockerfile, then return the tag func (p *DockerProvider) BuildImage(ctx context.Context, img ImageBuildInfo) (string, error) { - var buildOptions types.ImageBuildOptions - resp, err := backoff.RetryNotifyWithData( - func() (types.ImageBuildResponse, error) { - var err error - buildOptions, err = img.BuildOptions() - if err != nil { - return types.ImageBuildResponse{}, backoff.Permanent(fmt.Errorf("build options: %w", err)) - } - defer tryClose(buildOptions.Context) // release resources in any case - - resp, err := p.client.ImageBuild(ctx, buildOptions.Context, buildOptions) - if err != nil { - if isPermanentClientError(err) { - return types.ImageBuildResponse{}, backoff.Permanent(fmt.Errorf("build image: %w", err)) - } - return types.ImageBuildResponse{}, err - } - defer p.Close() - - return resp, nil - }, - backoff.WithContext(backoff.NewExponentialBackOff(), ctx), - func(err error, _ time.Duration) { - p.Logger.Printf("Failed to build image: %s, will retry", err) - }, - ) - if err != nil { - return "", err // Error is already wrapped. - } - defer resp.Body.Close() - - output := img.BuildLogWriter() - - // Always process the output, even if it is not printed - // to ensure that errors during the build process are - // correctly handled. - termFd, isTerm := term.GetFdInfo(output) - if err = jsonmessage.DisplayJSONMessagesStream(resp.Body, output, termFd, isTerm, nil); err != nil { - return "", fmt.Errorf("build image: %w", err) - } - - // the first tag is the one we want - return buildOptions.Tags[0], nil + return tcimage.Build(ctx, img) } // CreateContainer fulfils a request for a container without starting it @@ -1066,7 +1024,7 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque } // always append the hub substitutor after the user-defined ones - req.ImageSubstitutors = append(req.ImageSubstitutors, newPrependHubRegistry(p.config.HubImageNamePrefix)) + req.ImageSubstitutors = append(req.ImageSubstitutors, tcimage.NewPrependHubRegistry(p.config.HubImageNamePrefix)) var platform *specs.Platform @@ -1084,7 +1042,7 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque return nil, err } - imageName, err = p.BuildImage(ctx, &req) + imageName, err = tcimage.Build(ctx, &req) if err != nil { return nil, err } @@ -1288,7 +1246,7 @@ func (p *DockerProvider) waitContainerCreation(ctx context.Context, name string) func() (*types.Container, error) { c, err := p.findContainerByName(ctx, name) if err != nil { - if !errdefs.IsNotFound(err) && isPermanentClientError(err) { + if !errdefs.IsNotFound(err) && core.IsPermanentClientError(err) { return nil, backoff.Permanent(err) } return nil, err @@ -1405,7 +1363,7 @@ func (p *DockerProvider) attemptToPullImage(ctx context.Context, tag string, pul func() error { pull, err = p.client.ImagePull(ctx, tag, pullOpt) if err != nil { - if isPermanentClientError(err) { + if core.IsPermanentClientError(err) { return backoff.Permanent(err) } return err @@ -1726,78 +1684,20 @@ func (p *DockerProvider) ContainerFromType(ctx context.Context, response types.C return ctr, nil } +// Deprecated: use testcontainers-go [image.List] instead // ListImages list images from the provider. If an image has multiple Tags, each tag is reported // individually with the same ID and same labels func (p *DockerProvider) ListImages(ctx context.Context) ([]ImageInfo, error) { - images := []ImageInfo{} - - imageList, err := p.client.ImageList(ctx, image.ListOptions{}) - if err != nil { - return images, fmt.Errorf("listing images %w", err) - } - - for _, img := range imageList { - for _, tag := range img.RepoTags { - images = append(images, ImageInfo{ID: img.ID, Name: tag}) - } - } - - return images, nil + return tcimage.List(ctx) } +// Deprecated: use testcontainers-go [image.SaveToTar] instead // SaveImages exports a list of images as an uncompressed tar func (p *DockerProvider) SaveImages(ctx context.Context, output string, images ...string) error { - outputFile, err := os.Create(output) - if err != nil { - return fmt.Errorf("opening output file %w", err) - } - defer func() { - _ = outputFile.Close() - }() - - imageReader, err := p.client.ImageSave(ctx, images) - if err != nil { - return fmt.Errorf("saving images %w", err) - } - defer func() { - _ = imageReader.Close() - }() - - // Attempt optimized readFrom, implemented in linux - _, err = outputFile.ReadFrom(imageReader) - if err != nil { - return fmt.Errorf("writing images to output %w", err) - } - - return nil + return tcimage.SaveToTar(ctx, output, images...) } // PullImage pulls image from registry func (p *DockerProvider) PullImage(ctx context.Context, img string) error { return p.attemptToPullImage(ctx, img, image.PullOptions{}) } - -var permanentClientErrors = []func(error) bool{ - errdefs.IsNotFound, - errdefs.IsInvalidParameter, - errdefs.IsUnauthorized, - errdefs.IsForbidden, - errdefs.IsNotImplemented, - errdefs.IsSystem, -} - -func isPermanentClientError(err error) bool { - for _, isErrFn := range permanentClientErrors { - if isErrFn(err) { - return true - } - } - return false -} - -func tryClose(r io.Reader) { - rc, ok := r.(io.Closer) - if ok { - _ = rc.Close() - } -} diff --git a/docker_test.go b/docker_test.go index e9a9dd82f2..2c9d3d0b34 100644 --- a/docker_test.go +++ b/docker_test.go @@ -15,16 +15,16 @@ import ( "testing" "time" - "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/strslice" - "github.com/docker/docker/client" "github.com/docker/docker/errdefs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + tcimage "github.com/testcontainers/testcontainers-go/image" "github.com/testcontainers/testcontainers-go/internal/core" + "github.com/testcontainers/testcontainers-go/internal/core/mock" "github.com/testcontainers/testcontainers-go/log" "github.com/testcontainers/testcontainers-go/wait" ) @@ -1994,111 +1994,6 @@ func TestImageBuiltFromDockerfile_KeepBuiltImage(t *testing.T) { } } -// errMockCli is a mock implementation of client.APIClient, which is handy for simulating -// error returns in retry scenarios. -type errMockCli struct { - client.APIClient - - err error - imageBuildCount int - containerListCount int - imagePullCount int -} - -func (f *errMockCli) ImageBuild(_ context.Context, _ io.Reader, _ types.ImageBuildOptions) (types.ImageBuildResponse, error) { - f.imageBuildCount++ - return types.ImageBuildResponse{Body: io.NopCloser(&bytes.Buffer{})}, f.err -} - -func (f *errMockCli) ContainerList(_ context.Context, _ container.ListOptions) ([]types.Container, error) { - f.containerListCount++ - return []types.Container{{}}, f.err -} - -func (f *errMockCli) ImagePull(_ context.Context, _ string, _ image.PullOptions) (io.ReadCloser, error) { - f.imagePullCount++ - return io.NopCloser(&bytes.Buffer{}), f.err -} - -func (f *errMockCli) Close() error { - return nil -} - -func TestDockerProvider_BuildImage_Retries(t *testing.T) { - tests := []struct { - name string - errReturned error - shouldRetry bool - }{ - { - name: "no retry on success", - errReturned: nil, - shouldRetry: false, - }, - { - name: "no retry when a resource is not found", - errReturned: errdefs.NotFound(errors.New("not available")), - shouldRetry: false, - }, - { - name: "no retry when parameters are invalid", - errReturned: errdefs.InvalidParameter(errors.New("invalid")), - shouldRetry: false, - }, - { - name: "no retry when resource access not authorized", - errReturned: errdefs.Unauthorized(errors.New("not authorized")), - shouldRetry: false, - }, - { - name: "no retry when resource access is forbidden", - errReturned: errdefs.Forbidden(errors.New("forbidden")), - shouldRetry: false, - }, - { - name: "no retry when not implemented by provider", - errReturned: errdefs.NotImplemented(errors.New("unknown method")), - shouldRetry: false, - }, - { - name: "no retry on system error", - errReturned: errdefs.System(errors.New("system error")), - shouldRetry: false, - }, - { - name: "retry on non-permanent error", - errReturned: errors.New("whoops"), - shouldRetry: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p, err := NewDockerProvider() - require.NoError(t, err) - m := &errMockCli{err: tt.errReturned} - p.client = m - - // give a chance to retry - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - _, err = p.BuildImage(ctx, &ContainerRequest{ - FromDockerfile: FromDockerfile{ - Context: filepath.Join(".", "testdata", "retry"), - }, - }) - if tt.errReturned != nil { - require.Error(t, err) - } else { - require.NoError(t, err) - } - - assert.Positive(t, m.imageBuildCount) - assert.Equal(t, tt.shouldRetry, m.imageBuildCount > 1) - }) - } -} - func TestDockerProvider_waitContainerCreation_retries(t *testing.T) { tests := []struct { name string @@ -2136,7 +2031,7 @@ func TestDockerProvider_waitContainerCreation_retries(t *testing.T) { t.Run(tt.name, func(t *testing.T) { p, err := NewDockerProvider() require.NoError(t, err) - m := &errMockCli{err: tt.errReturned} + m := mock.NewErrClient(tt.errReturned) p.client = m // give a chance to retry @@ -2144,8 +2039,8 @@ func TestDockerProvider_waitContainerCreation_retries(t *testing.T) { defer cancel() _, _ = p.waitContainerCreation(ctx, "someID") - assert.Positive(t, m.containerListCount) - assert.Equal(t, tt.shouldRetry, m.containerListCount > 1) + assert.Positive(t, m.ContainerListCount()) + assert.Equal(t, tt.shouldRetry, m.ContainerListCount() > 1) }) } } @@ -2197,7 +2092,7 @@ func TestDockerProvider_attemptToPullImage_retries(t *testing.T) { t.Run(tt.name, func(t *testing.T) { p, err := NewDockerProvider() require.NoError(t, err) - m := &errMockCli{err: tt.errReturned} + m := mock.NewErrClient(tt.errReturned) p.client = m // give a chance to retry @@ -2205,8 +2100,8 @@ func TestDockerProvider_attemptToPullImage_retries(t *testing.T) { defer cancel() _ = p.attemptToPullImage(ctx, "someTag", image.PullOptions{}) - assert.Positive(t, m.imagePullCount) - assert.Equal(t, tt.shouldRetry, m.imagePullCount > 1) + assert.Positive(t, m.ImagePullCount()) + assert.Equal(t, tt.shouldRetry, m.ImagePullCount() > 1) }) } } @@ -2218,7 +2113,7 @@ func TestCustomPrefixTrailingSlashIsProperlyRemovedIfPresent(t *testing.T) { ctx := context.Background() req := ContainerRequest{ Image: dockerImage, - ImageSubstitutors: []ImageSubstitutor{newPrependHubRegistry(hubPrefixWithTrailingSlash)}, + ImageSubstitutors: []tcimage.Substitutor{tcimage.NewPrependHubRegistry(hubPrefixWithTrailingSlash)}, } c, err := GenericContainer(ctx, GenericContainerRequest{ diff --git a/docs/features/common_functional_options.md b/docs/features/common_functional_options.md index c7a0a334df..040669a767 100644 --- a/docs/features/common_functional_options.md +++ b/docs/features/common_functional_options.md @@ -6,11 +6,11 @@ In more locked down / secured environments, it can be problematic to pull images An image name substitutor converts a Docker image name, as may be specified in code, to an alternative name. This is intended to provide a way to override image names, for example to enforce pulling of images from a private registry. -_Testcontainers for Go_ exposes an interface to perform this operation: `ImageSubstitutor`, and a No-operation implementation to be used as reference for custom implementations: +_Testcontainers for Go_ exposes an interface in the `image` package to perform this operation: `image.Substitutor`, and a No-operation implementation to be used as reference for custom implementations, exposed as `image.NoopSubstitutor`: -[Image Substitutor Interface](../../options.go) inside_block:imageSubstitutor -[Noop Image Substitutor](../../container_test.go) inside_block:noopImageSubstitutor +[Image Substitutor Interface](../../image/substitutors.go) inside_block:imageSubstitutor +[Noop Image Substitutor](../../image/substitutors.go) inside_block:noopImageSubstitutor Using the `WithImageSubstitutors` options, you could define your own substitutions to the container images. E.g. adding a prefix to the images so that they can be pulled from a Docker registry other than Docker Hub. This is the usual mechanism for using Docker image proxies, caches, etc. diff --git a/docs/features/image_name_substitution.md b/docs/features/image_name_substitution.md index 4272d35c1e..e5a367e62f 100644 --- a/docs/features/image_name_substitution.md +++ b/docs/features/image_name_substitution.md @@ -65,14 +65,14 @@ In this case, image name references in code are **unchanged**. i.e. you would le You can implement a custom image name substitutor by: -* implementing the `ImageNameSubstitutor` interface, exposed by the `testcontainers` package. +* implementing the `image.Substitutor` interface, exposed by the `image` package. * configuring _Testcontainers for Go_ to use your custom implementation, defined at the `ContainerRequest` level. The following is an example image substitutor implementation prepending the `registry.hub.docker.com/library/` prefix, used in the tests: -[Image Substitutor Interface](../../options.go) inside_block:imageSubstitutor -[Docker prefix Image Substitutor](../../container_test.go) inside_block:dockerImageSubstitutor +[Image Substitutor Interface](../../image/substitutors.go) inside_block:imageSubstitutor +[Docker prefix Image Substitutor](../../image/substitutors.go) inside_block:dockerImageSubstitutor [Applying the substitutor](../../container_test.go) inside_block:applyImageSubstitutors diff --git a/from_dockerfile_test.go b/from_dockerfile_test.go index 99ce291c65..3a8c8501d1 100644 --- a/from_dockerfile_test.go +++ b/from_dockerfile_test.go @@ -13,19 +13,15 @@ import ( "github.com/docker/docker/api/types/image" "github.com/stretchr/testify/require" + tcimage "github.com/testcontainers/testcontainers-go/image" + "github.com/testcontainers/testcontainers-go/internal/core" "github.com/testcontainers/testcontainers-go/log" ) func TestBuildImageFromDockerfile(t *testing.T) { - provider, err := NewDockerProvider() - require.NoError(t, err) - defer provider.Close() - - cli := provider.Client() - ctx := context.Background() - tag, err := provider.BuildImage(ctx, &ContainerRequest{ + tag, err := tcimage.Build(ctx, &ContainerRequest{ // fromDockerfileIncludingRepo { FromDockerfile: FromDockerfile{ Context: "testdata", @@ -38,6 +34,10 @@ func TestBuildImageFromDockerfile(t *testing.T) { require.NoError(t, err) require.Equal(t, "test-repo:test-tag", tag) + cli, err := core.NewClient(context.Background()) + require.NoError(t, err) + defer cli.Close() + _, _, err = cli.ImageInspectWithRaw(ctx, tag) require.NoError(t, err) @@ -51,15 +51,9 @@ func TestBuildImageFromDockerfile(t *testing.T) { } func TestBuildImageFromDockerfile_NoRepo(t *testing.T) { - provider, err := NewDockerProvider() - require.NoError(t, err) - defer provider.Close() - - cli := provider.Client() - ctx := context.Background() - tag, err := provider.BuildImage(ctx, &ContainerRequest{ + tag, err := tcimage.Build(ctx, &ContainerRequest{ FromDockerfile: FromDockerfile{ Context: "testdata", Dockerfile: "echo.Dockerfile", @@ -69,6 +63,10 @@ func TestBuildImageFromDockerfile_NoRepo(t *testing.T) { require.NoError(t, err) require.True(t, strings.HasPrefix(tag, "test-repo:")) + cli, err := core.NewClient(context.Background()) + require.NoError(t, err) + defer cli.Close() + _, _, err = cli.ImageInspectWithRaw(ctx, tag) require.NoError(t, err) @@ -104,15 +102,9 @@ func TestBuildImageFromDockerfile_BuildError(t *testing.T) { } func TestBuildImageFromDockerfile_NoTag(t *testing.T) { - provider, err := NewDockerProvider() - require.NoError(t, err) - defer provider.Close() - - cli := provider.Client() - ctx := context.Background() - tag, err := provider.BuildImage(ctx, &ContainerRequest{ + tag, err := tcimage.Build(ctx, &ContainerRequest{ FromDockerfile: FromDockerfile{ Context: "testdata", Dockerfile: "echo.Dockerfile", @@ -122,6 +114,10 @@ func TestBuildImageFromDockerfile_NoTag(t *testing.T) { require.NoError(t, err) require.True(t, strings.HasSuffix(tag, ":test-tag")) + cli, err := core.NewClient(context.Background()) + require.NoError(t, err) + defer cli.Close() + _, _, err = cli.ImageInspectWithRaw(ctx, tag) require.NoError(t, err) diff --git a/image.go b/image.go index 4816fb7894..7997f02492 100644 --- a/image.go +++ b/image.go @@ -2,17 +2,17 @@ package testcontainers import ( "context" + + "github.com/testcontainers/testcontainers-go/image" ) +// Deprecated: use testcontainers-go [image.Info] instead // ImageInfo represents summary information of an image -type ImageInfo struct { - ID string - Name string -} +type ImageInfo = image.Info // ImageProvider allows manipulating images type ImageProvider interface { - ListImages(context.Context) ([]ImageInfo, error) - SaveImages(context.Context, string, ...string) error + ListImages(context.Context) ([]ImageInfo, error) // Deprecated: use testcontainers-go [image.List] instead + SaveImages(context.Context, string, ...string) error // Deprecated: use testcontainers-go [image.SaveToTar] instead PullImage(context.Context, string) error } diff --git a/image/build.go b/image/build.go new file mode 100644 index 0000000000..e62baf6781 --- /dev/null +++ b/image/build.go @@ -0,0 +1,97 @@ +package image + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/registry" + "github.com/docker/docker/client" + "github.com/docker/docker/pkg/jsonmessage" + "github.com/moby/term" + + "github.com/testcontainers/testcontainers-go/internal/core" + "github.com/testcontainers/testcontainers-go/log" +) + +// Builder defines what is needed to build an image +type Builder interface { + BuildOptions() (types.ImageBuildOptions, error) // converts the Builder to a types.ImageBuildOptions + GetContext() (io.Reader, error) // the path to the build context + GetDockerfile() string // the relative path to the Dockerfile, including the file itself + GetRepo() string // get repo label for image + GetTag() string // get tag label for image + BuildLogWriter() io.Writer // for output of build log, use io.Discard to disable the output + ShouldBuildImage() bool // return true if the image needs to be built + GetBuildArgs() map[string]*string // return the environment args used to build the Dockerfile + GetAuthConfigs() map[string]registry.AuthConfig // Deprecated. Testcontainers will detect registry credentials automatically. Return the auth configs to be able to pull from an authenticated docker registry +} + +// Build build an image from the given Builder, using the default docker client. +// It returns the first tag of the image. +func Build(ctx context.Context, img Builder) (string, error) { + cli, err := core.NewClient(ctx) + if err != nil { + return "", fmt.Errorf("new client: %w", err) + } + defer cli.Close() + + return buildWithClient(ctx, cli, img) +} + +// buildWithClient build and image from context and Dockerfile, then return the first tag of the image. +// The caller is responsible for closing the docker client. +func buildWithClient(ctx context.Context, dockerClient client.APIClient, img Builder) (string, error) { + var buildOptions types.ImageBuildOptions + resp, err := backoff.RetryNotifyWithData( + func() (types.ImageBuildResponse, error) { + var err error + buildOptions, err = img.BuildOptions() + if err != nil { + return types.ImageBuildResponse{}, backoff.Permanent(fmt.Errorf("build options: %w", err)) + } + defer tryClose(buildOptions.Context) // release resources in any case + + resp, err := dockerClient.ImageBuild(ctx, buildOptions.Context, buildOptions) + if err != nil { + if core.IsPermanentClientError(err) { + return types.ImageBuildResponse{}, backoff.Permanent(fmt.Errorf("build image: %w", err)) + } + return types.ImageBuildResponse{}, err + } + + return resp, nil + }, + backoff.WithContext(backoff.NewExponentialBackOff(), ctx), + func(err error, _ time.Duration) { + log.Printf("Failed to build image: %s, will retry", err) + }, + ) + if err != nil { + return "", err // Error is already wrapped. + } + defer resp.Body.Close() + + output := img.BuildLogWriter() + + // Always process the output, even if it is not printed + // to ensure that errors during the build process are + // correctly handled. + termFd, isTerm := term.GetFdInfo(output) + if err = jsonmessage.DisplayJSONMessagesStream(resp.Body, output, termFd, isTerm, nil); err != nil { + return "", fmt.Errorf("build image: %w", err) + } + + // the first tag is the one we want + return buildOptions.Tags[0], nil +} + +func tryClose(r io.Reader) { + rc, ok := r.(io.Closer) + if ok { + _ = rc.Close() + } +} diff --git a/image/build_test.go b/image/build_test.go new file mode 100644 index 0000000000..d0d3236691 --- /dev/null +++ b/image/build_test.go @@ -0,0 +1,137 @@ +package image + +import ( + "context" + "errors" + "io" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/registry" + "github.com/docker/docker/errdefs" + "github.com/stretchr/testify/require" + + "github.com/testcontainers/testcontainers-go/internal/core/mock" +) + +// testBuilder is a test helper that implements the Builder interface. +type testBuilder struct { + Builder +} + +// BuildOptions returns a types.ImageBuildOptions for the testBuilder, +// including one single tag in the Tags field. +func (t *testBuilder) BuildOptions() (types.ImageBuildOptions, error) { + return types.ImageBuildOptions{ + Tags: []string{"test"}, // needed because, the Build function returns the first tag + }, nil +} + +// GetContext returns nil, nil for the testBuilder. +func (t *testBuilder) GetContext() (io.Reader, error) { + return nil, nil +} + +// GetDockerfile returns an empty string for the testBuilder. +func (t *testBuilder) GetDockerfile() string { + return "" +} + +// GetRepo returns an empty string for the testBuilder. +func (t *testBuilder) GetRepo() string { + return "" +} + +// GetTag returns an empty string for the testBuilder. +func (t *testBuilder) GetTag() string { + return "" +} + +// BuildLogWriter returns io.Discard for the testBuilder. +func (t *testBuilder) BuildLogWriter() io.Writer { + return io.Discard +} + +// ShouldBuildImage returns true for the testBuilder. +func (t *testBuilder) ShouldBuildImage() bool { + return true +} + +// GetBuildArgs returns nil for the testBuilder. +func (t *testBuilder) GetBuildArgs() map[string]*string { + return nil +} + +// GetAuthConfigs returns nil for the testBuilder. +func (t *testBuilder) GetAuthConfigs() map[string]registry.AuthConfig { + return nil +} + +func TestDockerProvider_BuildImage_Retries(t *testing.T) { + tests := []struct { + name string + errReturned error + shouldRetry bool + }{ + { + name: "success/no-retry", + errReturned: nil, + shouldRetry: false, + }, + { + name: "resource-not-found/no-retry", + errReturned: errdefs.NotFound(errors.New("not available")), + shouldRetry: false, + }, + { + name: "invalid-parameters/no-retry", + errReturned: errdefs.InvalidParameter(errors.New("invalid")), + shouldRetry: false, + }, + { + name: "resource-access-unauthorized/no-retry", + errReturned: errdefs.Unauthorized(errors.New("not authorized")), + shouldRetry: false, + }, + { + name: "resource-access-forbidden/no-retry", + errReturned: errdefs.Forbidden(errors.New("forbidden")), + shouldRetry: false, + }, + { + name: "not-implemented-by-provider/no-retry", + errReturned: errdefs.NotImplemented(errors.New("unknown method")), + shouldRetry: false, + }, + { + name: "system-error/no-retry", + errReturned: errdefs.System(errors.New("system error")), + shouldRetry: false, + }, + { + name: "non-permanent-error/retry", + errReturned: errors.New("whoops"), + shouldRetry: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockCli := mock.NewErrClient(tt.errReturned) + + // give a chance to retry + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + _, err := buildWithClient(ctx, mockCli, &testBuilder{}) + if tt.errReturned != nil { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + require.Positive(t, mockCli.ImageBuildCount()) + require.Equal(t, tt.shouldRetry, mockCli.ImageBuildCount() > 1) + }) + } +} diff --git a/image/dockerignore.go b/image/dockerignore.go new file mode 100644 index 0000000000..80f0fd0895 --- /dev/null +++ b/image/dockerignore.go @@ -0,0 +1,29 @@ +package image + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/moby/patternmatcher/ignorefile" +) + +// ParseDockerIgnore returns if the file exists, the excluded files and an error if any +func ParseDockerIgnore(targetDir string) (bool, []string, error) { + // based on https://github.com/docker/cli/blob/master/cli/command/image/build/dockerignore.go#L14 + fileLocation := filepath.Join(targetDir, ".dockerignore") + var excluded []string + exists := false + if f, openErr := os.Open(fileLocation); openErr == nil { + defer f.Close() + + exists = true + + var err error + excluded, err = ignorefile.ReadAll(f) + if err != nil { + return true, excluded, fmt.Errorf("error reading .dockerignore: %w", err) + } + } + return exists, excluded, nil +} diff --git a/image/dockerignore_test.go b/image/dockerignore_test.go new file mode 100644 index 0000000000..e39dffb9ae --- /dev/null +++ b/image/dockerignore_test.go @@ -0,0 +1,30 @@ +package image + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseDockerIgnore(t *testing.T) { + assertions := func(t *testing.T, filePath string, exists bool, expectedErr error, expectedExcluded []string) { + t.Helper() + + ok, excluded, err := ParseDockerIgnore(filePath) + require.Equal(t, exists, ok) + require.Equal(t, expectedErr, err) + require.Equal(t, expectedExcluded, excluded) + } + + t.Run("file-exists", func(t *testing.T) { + assertions(t, "./testdata/dockerignore", true, nil, []string{"vendor", "foo", "bar"}) + }) + + t.Run("file-exists-including-comments", func(t *testing.T) { + assertions(t, "./testdata", true, nil, []string{"Dockerfile", "echo.Dockerfile"}) + }) + + t.Run("file-does-not-exists", func(t *testing.T) { + assertions(t, "./testdata/data", false, nil, nil) + }) +} diff --git a/image/info.go b/image/info.go new file mode 100644 index 0000000000..ccb33350fb --- /dev/null +++ b/image/info.go @@ -0,0 +1,7 @@ +package image + +// Info represents a summary information of an image +type Info struct { + ID string + Name string +} diff --git a/image/list.go b/image/list.go new file mode 100644 index 0000000000..ae444d968f --- /dev/null +++ b/image/list.go @@ -0,0 +1,35 @@ +package image + +import ( + "context" + "fmt" + + "github.com/docker/docker/api/types/image" + + "github.com/testcontainers/testcontainers-go/internal/core" +) + +// List list images from the provider. If an image has multiple Tags, each tag is reported +// individually with the same ID and same labels +func List(ctx context.Context) ([]Info, error) { + images := []Info{} + + cli, err := core.NewClient(ctx) + if err != nil { + return images, err + } + defer cli.Close() + + imageList, err := cli.ImageList(ctx, image.ListOptions{}) + if err != nil { + return images, fmt.Errorf("listing images %w", err) + } + + for _, img := range imageList { + for _, tag := range img.RepoTags { + images = append(images, Info{ID: img.ID, Name: tag}) + } + } + + return images, nil +} diff --git a/image/save.go b/image/save.go new file mode 100644 index 0000000000..04fddf31b6 --- /dev/null +++ b/image/save.go @@ -0,0 +1,42 @@ +package image + +import ( + "context" + "fmt" + "os" + + "github.com/testcontainers/testcontainers-go/internal/core" +) + +// SaveToTar exports a list of images as an uncompressed tar +func SaveToTar(ctx context.Context, output string, images ...string) error { + outputFile, err := os.Create(output) + if err != nil { + return fmt.Errorf("opening output file %w", err) + } + defer func() { + _ = outputFile.Close() + }() + + cli, err := core.NewClient(ctx) + if err != nil { + return err + } + defer cli.Close() + + imageReader, err := cli.ImageSave(ctx, images) + if err != nil { + return fmt.Errorf("saving images %w", err) + } + defer func() { + _ = imageReader.Close() + }() + + // Attempt optimized readFrom, implemented in linux + _, err = outputFile.ReadFrom(imageReader) + if err != nil { + return fmt.Errorf("writing images to output %w", err) + } + + return nil +} diff --git a/image/substitutors.go b/image/substitutors.go new file mode 100644 index 0000000000..180dd9fd41 --- /dev/null +++ b/image/substitutors.go @@ -0,0 +1,149 @@ +package image + +import ( + "fmt" + "net/url" + + "github.com/testcontainers/testcontainers-go/internal/config" + "github.com/testcontainers/testcontainers-go/internal/core" +) + +// imageSubstitutor { + +// Substitutor represents a way to substitute container image names +type Substitutor interface { + // Description returns the name of the type and a short description of how it modifies the image. + // Useful to be printed in logs + Description() string + Substitute(image string) (string, error) +} + +// } + +// CustomHubSubstitutor represents a way to substitute the hub of an image with a custom one, +// using provided value with respect to the HubImageNamePrefix configuration value. +type CustomHubSubstitutor struct { + hub string +} + +// NewCustomHubSubstitutor creates a new CustomHubSubstitutor +func NewCustomHubSubstitutor(hub string) CustomHubSubstitutor { + return CustomHubSubstitutor{ + hub: hub, + } +} + +// Description returns the name of the type and a short description of how it modifies the image. +func (c CustomHubSubstitutor) Description() string { + return fmt.Sprintf("CustomHubSubstitutor (replaces hub with %s)", c.hub) +} + +// Substitute replaces the hub of the image with the provided one, with certain conditions: +// - if the hub is empty, the image is returned as is. +// - if the image already contains a registry, the image is returned as is. +// - if the HubImageNamePrefix configuration value is set in the Testcontainers configuration, the image is returned as is. +func (c CustomHubSubstitutor) Substitute(image string) (string, error) { + registry := core.ExtractRegistry(image, "") + cfg := config.Read() + + exclusions := []func() bool{ + func() bool { return c.hub == "" }, + func() bool { return registry != "" }, + func() bool { return cfg.HubImageNamePrefix != "" }, + } + + for _, exclusion := range exclusions { + if exclusion() { + return image, nil + } + } + + result, err := url.JoinPath(c.hub, image) + if err != nil { + return "", err + } + + return result, nil +} + +// PrependHubRegistry represents a way to prepend a custom Hub registry to the image name, +// using the HubImageNamePrefix configuration value +type PrependHubRegistry struct { + prefix string +} + +// NewPrependHubRegistry creates a new PrependHubRegistry +func NewPrependHubRegistry(hubPrefix string) PrependHubRegistry { + return PrependHubRegistry{ + prefix: hubPrefix, + } +} + +// Description returns the name of the type and a short description of how it modifies the image. +func (p PrependHubRegistry) Description() string { + return fmt.Sprintf("HubImageSubstitutor (prepends %s)", p.prefix) +} + +// Substitute prepends the Hub prefix to the image name, with certain conditions: +// - if the prefix is empty, the image is returned as is. +// - if the image is a non-hub image (e.g. where another registry is set), the image is returned as is. +// - if the image is a Docker Hub image where the hub registry is explicitly part of the name +// (i.e. anything with a registry.hub.docker.com host part), the image is returned as is. +func (p PrependHubRegistry) Substitute(image string) (string, error) { + registry := core.ExtractRegistry(image, "") + + // add the exclusions in the right order + exclusions := []func() bool{ + func() bool { return p.prefix == "" }, // no prefix set at the configuration level + func() bool { return registry != "" }, // non-hub image + func() bool { return registry == "docker.io" }, // explicitly including docker.io + func() bool { return registry == "registry.hub.docker.com" }, // explicitly including registry.hub.docker.com + } + + for _, exclusion := range exclusions { + if exclusion() { + return image, nil + } + } + + result, err := url.JoinPath(p.prefix, image) + if err != nil { + return "", err + } + + return result, nil +} + +// noopImageSubstitutor { +// NoopSubstitutor is a Substitutor that returns the original image, without any change. +type NoopSubstitutor struct{} + +// Description returns a description of what is expected from this Substitutor, +// which is used in logs. +func (s NoopSubstitutor) Description() string { + return "NoopSubstitutor (noop)" +} + +// Substitute returns the original image, without any change +func (s NoopSubstitutor) Substitute(image string) (string, error) { + return image, nil +} + +// } + +// dockerImageSubstitutor { +// DockerSubstitutor is a Substitutor that prepends the registry.hub.docker.com prefix to the image name. +type DockerSubstitutor struct{} + +// Description returns a description of what is expected from this Substitutor, +// which is used in logs. +func (s DockerSubstitutor) Description() string { + return "DockerSubstitutor (prepends registry.hub.docker.com)" +} + +// Substitute prepends the registry.hub.docker.com prefix to the image name. +func (s DockerSubstitutor) Substitute(image string) (string, error) { + return "registry.hub.docker.com/library/" + image, nil +} + +// } diff --git a/image/substitutors_test.go b/image/substitutors_test.go new file mode 100644 index 0000000000..f6fc9f6258 --- /dev/null +++ b/image/substitutors_test.go @@ -0,0 +1,98 @@ +package image + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/testcontainers/testcontainers-go/internal/config" +) + +func TestCustomHubSubstitutor(t *testing.T) { + t.Run("prepend", func(t *testing.T) { + s := NewCustomHubSubstitutor("quay.io") + + img, err := s.Substitute("foo/foo:latest") + require.NoError(t, err) + + require.Equalf(t, "quay.io/foo/foo:latest", img, "expected quay.io/foo/foo:latest, got %s", img) + }) + t.Run("not-prepend-as-prefix-exists", func(t *testing.T) { + s := NewCustomHubSubstitutor("quay.io") + + img, err := s.Substitute("quay.io/foo/foo:latest") + require.NoError(t, err) + + require.Equalf(t, "quay.io/foo/foo:latest", img, "expected quay.io/foo/foo:latest, got %s", img) + }) + t.Run("not-prepend-as-config-hub-prefix-exists", func(t *testing.T) { + t.Cleanup(config.Reset) + config.Reset() + t.Setenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX", "registry.mycompany.com/mirror") + s := NewCustomHubSubstitutor("quay.io") + + img, err := s.Substitute("foo/foo:latest") + require.NoError(t, err) + + require.Equalf(t, "foo/foo:latest", img, "expected foo/foo:latest, got %s", img) + }) +} + +func TestPrependHubRegistrySubstitutor(t *testing.T) { + t.Run("prepend", func(t *testing.T) { + t.Run("plain-image", func(t *testing.T) { + s := NewPrependHubRegistry("my-registry") + + img, err := s.Substitute("foo:latest") + require.NoError(t, err) + + require.Equalf(t, "my-registry/foo:latest", img, "expected my-registry/foo, got %s", img) + }) + t.Run("image-with-user", func(t *testing.T) { + s := NewPrependHubRegistry("my-registry") + + img, err := s.Substitute("user/foo:latest") + require.NoError(t, err) + + require.Equalf(t, "my-registry/user/foo:latest", img, "expected my-registry/foo, got %s", img) + }) + + t.Run("image-with-organization-and-user", func(t *testing.T) { + s := NewPrependHubRegistry("my-registry") + + img, err := s.Substitute("org/user/foo:latest") + require.NoError(t, err) + + require.Equalf(t, "my-registry/org/user/foo:latest", img, "expected my-registry/org/foo:latest, got %s", img) + }) + }) + + t.Run("not-prepend", func(t *testing.T) { + t.Run("non-hub-image", func(t *testing.T) { + s := NewPrependHubRegistry("my-registry") + + img, err := s.Substitute("quay.io/foo:latest") + require.NoError(t, err) + + require.Equalf(t, "quay.io/foo:latest", img, "expected quay.io/foo:latest, got %s", img) + }) + + t.Run("prefix-is-docker-io", func(t *testing.T) { + s := NewPrependHubRegistry("my-registry") + + img, err := s.Substitute("docker.io/foo:latest") + require.NoError(t, err) + + require.Equalf(t, "docker.io/foo:latest", img, "expected docker.io/foo:latest, got %s", img) + }) + + t.Run("prefix-is-registry-hub-docker-com", func(t *testing.T) { + s := NewPrependHubRegistry("my-registry") + + img, err := s.Substitute("registry.hub.docker.com/foo:latest") + require.NoError(t, err) + + require.Equalf(t, "registry.hub.docker.com/foo:latest", img, "expected registry.hub.docker.com/foo:latest, got %s", img) + }) + }) +} diff --git a/image/testdata/.dockerignore b/image/testdata/.dockerignore new file mode 100644 index 0000000000..5adb76c74c --- /dev/null +++ b/image/testdata/.dockerignore @@ -0,0 +1,2 @@ +Dockerfile +echo.Dockerfile diff --git a/image/testdata/dockerignore/.dockerignore b/image/testdata/dockerignore/.dockerignore new file mode 100644 index 0000000000..1afdf210cf --- /dev/null +++ b/image/testdata/dockerignore/.dockerignore @@ -0,0 +1,5 @@ +# .dockerignore +# https://docs.docker.com/build/building/context/#dockerignore-files +vendor +foo +bar diff --git a/image_substitutors_test.go b/image_substitutors_test.go index c9d6aee244..d37ee83728 100644 --- a/image_substitutors_test.go +++ b/image_substitutors_test.go @@ -2,100 +2,94 @@ package testcontainers import ( "context" + "errors" "testing" "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go/image" "github.com/testcontainers/testcontainers-go/internal/config" ) -func TestCustomHubSubstitutor(t *testing.T) { - t.Run("should substitute the image with the provided one", func(t *testing.T) { - s := NewCustomHubSubstitutor("quay.io") +// Deprecated: use testcontainers-go [image.NoopSubstitutor] instead +type NoopImageSubstitutor = image.NoopSubstitutor - img, err := s.Substitute("foo/foo:latest") - require.NoError(t, err) - - require.Equalf(t, "quay.io/foo/foo:latest", img, "expected quay.io/foo/foo:latest, got %s", img) - }) - t.Run("should not substitute the image if it is already using the provided hub", func(t *testing.T) { - s := NewCustomHubSubstitutor("quay.io") +// errorSubstitutor is a Substitutor that returns an error +type errorSubstitutor struct{} - img, err := s.Substitute("quay.io/foo/foo:latest") - require.NoError(t, err) +var errSubstitution = errors.New("substitution error") - require.Equalf(t, "quay.io/foo/foo:latest", img, "expected quay.io/foo/foo:latest, got %s", img) - }) - t.Run("should not substitute the image if hub image name prefix config exist", func(t *testing.T) { - t.Cleanup(config.Reset) - config.Reset() - t.Setenv("TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX", "registry.mycompany.com/mirror") - s := NewCustomHubSubstitutor("quay.io") - - img, err := s.Substitute("foo/foo:latest") - require.NoError(t, err) - - require.Equalf(t, "foo/foo:latest", img, "expected foo/foo:latest, got %s", img) - }) +// Description returns a description of what is expected from this Substitutor, +// which is used in logs. +func (s errorSubstitutor) Description() string { + return "errorSubstitutor" } -func TestPrependHubRegistrySubstitutor(t *testing.T) { - t.Run("should prepend the hub registry to images from Docker Hub", func(t *testing.T) { - t.Run("plain image", func(t *testing.T) { - s := newPrependHubRegistry("my-registry") - - img, err := s.Substitute("foo:latest") - require.NoError(t, err) - - require.Equalf(t, "my-registry/foo:latest", img, "expected my-registry/foo, got %s", img) - }) - t.Run("image with user", func(t *testing.T) { - s := newPrependHubRegistry("my-registry") - - img, err := s.Substitute("user/foo:latest") - require.NoError(t, err) - - require.Equalf(t, "my-registry/user/foo:latest", img, "expected my-registry/foo, got %s", img) - }) - - t.Run("image with organization and user", func(t *testing.T) { - s := newPrependHubRegistry("my-registry") - - img, err := s.Substitute("org/user/foo:latest") - require.NoError(t, err) - - require.Equalf(t, "my-registry/org/user/foo:latest", img, "expected my-registry/org/foo:latest, got %s", img) - }) - }) - - t.Run("should not prepend the hub registry to the image name", func(t *testing.T) { - t.Run("non-hub image", func(t *testing.T) { - s := newPrependHubRegistry("my-registry") - - img, err := s.Substitute("quay.io/foo:latest") - require.NoError(t, err) - - require.Equalf(t, "quay.io/foo:latest", img, "expected quay.io/foo:latest, got %s", img) - }) - - t.Run("explicitly including registry.hub.docker.com/library", func(t *testing.T) { - s := newPrependHubRegistry("my-registry") - - img, err := s.Substitute("registry.hub.docker.com/library/foo:latest") - require.NoError(t, err) +// Substitute returns the original image, but returns an error +func (s errorSubstitutor) Substitute(image string) (string, error) { + return image, errSubstitution +} - require.Equalf(t, "registry.hub.docker.com/library/foo:latest", img, "expected registry.hub.docker.com/library/foo:latest, got %s", img) - }) +func TestImageSubstitutors(t *testing.T) { + tests := []struct { + name string + image string // must be a valid image, as the test will try to create a container from it + substitutors []image.Substitutor + expectedImage string + expectedError error + }{ + { + name: "no-substitutors", + image: "alpine", + expectedImage: "alpine", + }, + { + name: "noop-substitutor", + image: "alpine", + substitutors: []image.Substitutor{image.NoopSubstitutor{}}, + expectedImage: "alpine", + }, + { + name: "prepend-namespace", + image: "alpine", + substitutors: []image.Substitutor{image.DockerSubstitutor{}}, + expectedImage: "registry.hub.docker.com/library/alpine", + }, + { + name: "substitution-with-error", + image: "alpine", + substitutors: []image.Substitutor{errorSubstitutor{}}, + expectedImage: "alpine", + expectedError: errSubstitution, + }, + } - t.Run("explicitly including registry.hub.docker.com", func(t *testing.T) { - s := newPrependHubRegistry("my-registry") + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + req := ContainerRequest{ + Image: test.image, + ImageSubstitutors: test.substitutors, + } + + ctr, err := GenericContainer(ctx, GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + CleanupContainer(t, ctr) + if test.expectedError != nil { + require.ErrorIs(t, err, test.expectedError) + return + } - img, err := s.Substitute("registry.hub.docker.com/foo:latest") require.NoError(t, err) - require.Equalf(t, "registry.hub.docker.com/foo:latest", img, "expected registry.hub.docker.com/foo:latest, got %s", img) + // enforce the concrete type, as GenericContainer returns an interface, + // which will be changed in future implementations of the library + dockerContainer := ctr.(*DockerContainer) + require.Equal(t, test.expectedImage, dockerContainer.Image) }) - }) + } } func TestSubstituteBuiltImage(t *testing.T) { @@ -107,12 +101,12 @@ func TestSubstituteBuiltImage(t *testing.T) { Tag: "my-image", Repo: "my-repo", }, - ImageSubstitutors: []ImageSubstitutor{newPrependHubRegistry("my-registry")}, + ImageSubstitutors: []image.Substitutor{image.NewPrependHubRegistry("my-registry")}, }, Started: false, } - t.Run("should not use the properties prefix on built images", func(t *testing.T) { + t.Run("should-use-image-substitutors", func(t *testing.T) { config.Reset() c, err := GenericContainer(context.Background(), req) CleanupContainer(t, c) diff --git a/image_test.go b/image_test.go index b5a95640a8..b9a2a16fd5 100644 --- a/image_test.go +++ b/image_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go/image" "github.com/testcontainers/testcontainers-go/internal/core" ) @@ -29,7 +30,7 @@ func TestImageList(t *testing.T) { CleanupContainer(t, ctr) require.NoErrorf(t, err, "creating test container") - images, err := provider.ListImages(context.Background()) + images, err := image.List(context.Background()) require.NoErrorf(t, err, "listing images") require.NotEmptyf(t, images, "no images retrieved") @@ -63,7 +64,7 @@ func TestSaveImages(t *testing.T) { require.NoErrorf(t, err, "creating test container") output := filepath.Join(t.TempDir(), "images.tar") - err = provider.SaveImages(context.Background(), output, req.Image) + err = image.SaveToTar(context.Background(), output, req.Image) require.NoErrorf(t, err, "saving image %q", req.Image) info, err := os.Stat(output) diff --git a/internal/core/client_errors.go b/internal/core/client_errors.go new file mode 100644 index 0000000000..517c1e072c --- /dev/null +++ b/internal/core/client_errors.go @@ -0,0 +1,23 @@ +package core + +import "github.com/docker/docker/errdefs" + +var permanentClientErrors = []func(error) bool{ + errdefs.IsNotFound, + errdefs.IsInvalidParameter, + errdefs.IsUnauthorized, + errdefs.IsForbidden, + errdefs.IsNotImplemented, + errdefs.IsSystem, +} + +// IsPermanentClientError returns true if the error is a permanent client error +// from the Docker client. +func IsPermanentClientError(err error) bool { + for _, isErrFn := range permanentClientErrors { + if isErrFn(err) { + return true + } + } + return false +} diff --git a/internal/core/mock/client.go b/internal/core/mock/client.go new file mode 100644 index 0000000000..2b0c4639c0 --- /dev/null +++ b/internal/core/mock/client.go @@ -0,0 +1,66 @@ +package mock + +import ( + "bytes" + "context" + "io" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/image" + "github.com/docker/docker/client" +) + +// ErrClient is a mock implementation of client.APIClient, which is handy for simulating +// error returns in retry scenarios. +type ErrClient struct { + client.APIClient + + err error + imageBuildCount int + containerListCount int + imagePullCount int +} + +// NewErrClient returns a new ErrClient with the given error. +func NewErrClient(err error) *ErrClient { + return &ErrClient{err: err} +} + +// ImageBuildCount returns the number of times ImageBuild has been called. +func (f *ErrClient) ImageBuildCount() int { + return f.imageBuildCount +} + +// ContainerListCount returns the number of times ContainerList has been called. +func (f *ErrClient) ContainerListCount() int { + return f.containerListCount +} + +// ImagePullCount returns the number of times ImagePull has been called. +func (f *ErrClient) ImagePullCount() int { + return f.imagePullCount +} + +// ImageBuild returns a mock implementation of client.APIClient.ImageBuild. +func (f *ErrClient) ImageBuild(_ context.Context, _ io.Reader, _ types.ImageBuildOptions) (types.ImageBuildResponse, error) { + f.imageBuildCount++ + return types.ImageBuildResponse{Body: io.NopCloser(&bytes.Buffer{})}, f.err +} + +// ContainerList returns a mock implementation of client.APIClient.ContainerList. +func (f *ErrClient) ContainerList(_ context.Context, _ container.ListOptions) ([]types.Container, error) { + f.containerListCount++ + return []types.Container{{}}, f.err +} + +// ImagePull returns a mock implementation of client.APIClient.ImagePull. +func (f *ErrClient) ImagePull(_ context.Context, _ string, _ image.PullOptions) (io.ReadCloser, error) { + f.imagePullCount++ + return io.NopCloser(&bytes.Buffer{}), f.err +} + +// Close returns a mock implementation of client.APIClient.Close. +func (f *ErrClient) Close() error { + return nil +} diff --git a/modules/k3s/k3s.go b/modules/k3s/k3s.go index 3c8d4b57e5..17cd50fd37 100644 --- a/modules/k3s/k3s.go +++ b/modules/k3s/k3s.go @@ -13,6 +13,7 @@ import ( "gopkg.in/yaml.v3" "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/image" "github.com/testcontainers/testcontainers-go/log" "github.com/testcontainers/testcontainers-go/wait" ) @@ -202,11 +203,6 @@ func unmarshal(bytes []byte) (*KubeConfigValue, error) { // LoadImages loads images into the k3s container. func (c *K3sContainer) LoadImages(ctx context.Context, images ...string) error { - provider, err := testcontainers.ProviderDocker.GetProvider() - if err != nil { - return fmt.Errorf("getting docker provider %w", err) - } - // save image imagesTar, err := os.CreateTemp(os.TempDir(), "images*.tar") if err != nil { @@ -216,7 +212,7 @@ func (c *K3sContainer) LoadImages(ctx context.Context, images ...string) error { _ = os.Remove(imagesTar.Name()) }() - err = provider.SaveImages(context.Background(), imagesTar.Name(), images...) + err = image.SaveToTar(context.Background(), imagesTar.Name(), images...) if err != nil { return fmt.Errorf("saving images %w", err) } diff --git a/options.go b/options.go index f17de88ad6..ce6d322fa8 100644 --- a/options.go +++ b/options.go @@ -3,7 +3,6 @@ package testcontainers import ( "context" "fmt" - "net/url" "time" "dario.cat/mergo" @@ -11,7 +10,7 @@ import ( "github.com/docker/docker/api/types/network" tcexec "github.com/testcontainers/testcontainers-go/exec" - "github.com/testcontainers/testcontainers-go/internal/core" + "github.com/testcontainers/testcontainers-go/image" "github.com/testcontainers/testcontainers-go/wait" ) @@ -106,114 +105,23 @@ func WithImage(image string) CustomizeRequestOption { } } -// imageSubstitutor { - +// Deprecated: use testcontainers-go [image.Substitutor] instead // ImageSubstitutor represents a way to substitute container image names -type ImageSubstitutor interface { - // Description returns the name of the type and a short description of how it modifies the image. - // Useful to be printed in logs - Description() string - Substitute(image string) (string, error) -} - -// } +type ImageSubstitutor = image.Substitutor +// Deprecated: use testcontainers-go [image.CustomHubSubstitutor] instead // CustomHubSubstitutor represents a way to substitute the hub of an image with a custom one, // using provided value with respect to the HubImageNamePrefix configuration value. -type CustomHubSubstitutor struct { - hub string -} +type CustomHubSubstitutor = image.CustomHubSubstitutor +// Deprecated: use testcontainers-go [image.NewCustomHubSubstitutor] instead // NewCustomHubSubstitutor creates a new CustomHubSubstitutor func NewCustomHubSubstitutor(hub string) CustomHubSubstitutor { - return CustomHubSubstitutor{ - hub: hub, - } -} - -// Description returns the name of the type and a short description of how it modifies the image. -func (c CustomHubSubstitutor) Description() string { - return fmt.Sprintf("CustomHubSubstitutor (replaces hub with %s)", c.hub) -} - -// Substitute replaces the hub of the image with the provided one, with certain conditions: -// - if the hub is empty, the image is returned as is. -// - if the image already contains a registry, the image is returned as is. -// - if the HubImageNamePrefix configuration value is set, the image is returned as is. -func (c CustomHubSubstitutor) Substitute(image string) (string, error) { - registry := core.ExtractRegistry(image, "") - cfg := ReadConfig() - - exclusions := []func() bool{ - func() bool { return c.hub == "" }, - func() bool { return registry != "" }, - func() bool { return cfg.Config.HubImageNamePrefix != "" }, - } - - for _, exclusion := range exclusions { - if exclusion() { - return image, nil - } - } - - result, err := url.JoinPath(c.hub, image) - if err != nil { - return "", err - } - - return result, nil -} - -// prependHubRegistry represents a way to prepend a custom Hub registry to the image name, -// using the HubImageNamePrefix configuration value -type prependHubRegistry struct { - prefix string -} - -// newPrependHubRegistry creates a new prependHubRegistry -func newPrependHubRegistry(hubPrefix string) prependHubRegistry { - return prependHubRegistry{ - prefix: hubPrefix, - } -} - -// Description returns the name of the type and a short description of how it modifies the image. -func (p prependHubRegistry) Description() string { - return fmt.Sprintf("HubImageSubstitutor (prepends %s)", p.prefix) -} - -// Substitute prepends the Hub prefix to the image name, with certain conditions: -// - if the prefix is empty, the image is returned as is. -// - if the image is a non-hub image (e.g. where another registry is set), the image is returned as is. -// - if the image is a Docker Hub image where the hub registry is explicitly part of the name -// (i.e. anything with a registry.hub.docker.com host part), the image is returned as is. -func (p prependHubRegistry) Substitute(image string) (string, error) { - registry := core.ExtractRegistry(image, "") - - // add the exclusions in the right order - exclusions := []func() bool{ - func() bool { return p.prefix == "" }, // no prefix set at the configuration level - func() bool { return registry != "" }, // non-hub image - func() bool { return registry == "docker.io" }, // explicitly including docker.io - func() bool { return registry == "registry.hub.docker.com" }, // explicitly including registry.hub.docker.com - } - - for _, exclusion := range exclusions { - if exclusion() { - return image, nil - } - } - - result, err := url.JoinPath(p.prefix, image) - if err != nil { - return "", err - } - - return result, nil + return image.NewCustomHubSubstitutor(hub) } // WithImageSubstitutors sets the image substitutors for a container -func WithImageSubstitutors(fn ...ImageSubstitutor) CustomizeRequestOption { +func WithImageSubstitutors(fn ...image.Substitutor) CustomizeRequestOption { return func(req *GenericContainerRequest) error { req.ImageSubstitutors = fn diff --git a/testdata/retry/Dockerfile b/testdata/retry/Dockerfile deleted file mode 100644 index c35f1b5f5e..0000000000 --- a/testdata/retry/Dockerfile +++ /dev/null @@ -1 +0,0 @@ -FROM scratch