From ad06f2a639011f619eb629bad352064a124bb3a8 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Tue, 22 Apr 2025 09:56:54 +0200 Subject: [PATCH] docker(install): use undock to extract image Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- __tests__/docker/install.test.itg.ts | 28 ++++- __tests__/docker/install.test.ts | 4 + src/docker/install.ts | 159 +++++++++++++++--------- src/hubRepository.ts | 174 --------------------------- 4 files changed, 127 insertions(+), 238 deletions(-) delete mode 100644 src/hubRepository.ts diff --git a/__tests__/docker/install.test.itg.ts b/__tests__/docker/install.test.itg.ts index 6f651a70..e55353eb 100644 --- a/__tests__/docker/install.test.itg.ts +++ b/__tests__/docker/install.test.itg.ts @@ -14,17 +14,31 @@ * limitations under the License. */ -import {describe, test, expect} from '@jest/globals'; +import {beforeAll, describe, test, expect} from '@jest/globals'; import fs from 'fs'; import os from 'os'; import path from 'path'; import {Install, InstallSource, InstallSourceArchive, InstallSourceImage} from '../../src/docker/install'; import {Docker} from '../../src/docker/docker'; +import {Regctl} from '../../src/regclient/regctl'; +import {Install as RegclientInstall} from '../../src/regclient/install'; +import {Undock} from '../../src/undock/undock'; +import {Install as UndockInstall} from '../../src/undock/install'; import {Exec} from '../../src/exec'; const tmpDir = () => fs.mkdtempSync(path.join(process.env.TEMP || os.tmpdir(), 'docker-install-itg-')); +beforeAll(async () => { + const undockInstall = new UndockInstall(); + const undockBinPath = await undockInstall.download('v0.10.0', true); + await undockInstall.install(undockBinPath); + + const regclientInstall = new RegclientInstall(); + const regclientBinPath = await regclientInstall.download('v0.8.2', true); + await regclientInstall.install(regclientBinPath); +}, 100000); + describe('root', () => { // prettier-ignore test.each(getSources(true))( @@ -34,7 +48,9 @@ describe('root', () => { source: source, runDir: tmpDir(), contextName: 'foo', - daemonConfig: `{"debug":true,"features":{"containerd-snapshotter":true}}` + daemonConfig: `{"debug":true,"features":{"containerd-snapshotter":true}}`, + regctl: new Regctl(), + undock: new Undock() }); await expect(tryInstall(install)).resolves.not.toThrow(); }, 30 * 60 * 1000); @@ -54,7 +70,9 @@ describe('rootless', () => { runDir: tmpDir(), contextName: 'foo', daemonConfig: `{"debug":true}`, - rootless: true + rootless: true, + regctl: new Regctl(), + undock: new Undock() }); await expect( tryInstall(install, async () => { @@ -79,7 +97,9 @@ describe('tcp', () => { runDir: tmpDir(), contextName: 'foo', daemonConfig: `{"debug":true}`, - localTCPPort: 2378 + localTCPPort: 2378, + regctl: new Regctl(), + undock: new Undock() }); await expect( tryInstall(install, async () => { diff --git a/__tests__/docker/install.test.ts b/__tests__/docker/install.test.ts index e3f12948..3f149834 100644 --- a/__tests__/docker/install.test.ts +++ b/__tests__/docker/install.test.ts @@ -22,6 +22,8 @@ import * as rimraf from 'rimraf'; import osm = require('os'); import {Install, InstallSourceArchive, InstallSourceImage} from '../../src/docker/install'; +import {Regctl} from '../../src/regclient/regctl'; +import {Undock} from '../../src/undock/undock'; const tmpDir = fs.mkdtempSync(path.join(process.env.TEMP || os.tmpdir(), 'docker-install-')); @@ -64,6 +66,8 @@ describe('download', () => { const install = new Install({ source: source, runDir: tmpDir, + regctl: new Regctl(), + undock: new Undock() }); const toolPath = await install.download(); expect(fs.existsSync(toolPath)).toBe(true); diff --git a/src/docker/install.ts b/src/docker/install.ts index 620109a5..9f72dd52 100644 --- a/src/docker/install.ts +++ b/src/docker/install.ts @@ -28,11 +28,13 @@ import * as tc from '@actions/tool-cache'; import {Context} from '../context'; import {Docker} from './docker'; +import {Regctl} from '../regclient/regctl'; +import {Undock} from '../undock/undock'; import {Exec} from '../exec'; import {Util} from '../util'; import {limaYamlData, dockerServiceLogsPs1, setupDockerWinPs1} from './assets'; + import {GitHubRelease} from '../types/github'; -import {HubRepository} from '../hubRepository'; import {Image} from '../types/oci/config'; export interface InstallSourceImage { @@ -57,6 +59,9 @@ export interface InstallOpts { daemonConfig?: string; rootless?: boolean; localTCPPort?: number; + + regctl: Regctl; + undock: Undock; } interface LimaImage { @@ -72,6 +77,8 @@ export class Install { private readonly daemonConfig?: string; private readonly rootless: boolean; private readonly localTCPPort?: number; + private readonly regctl: Regctl; + private readonly undock: Undock; private _version: string | undefined; private _toolDir: string | undefined; @@ -91,36 +98,14 @@ export class Install { this.daemonConfig = opts.daemonConfig; this.rootless = opts.rootless || false; this.localTCPPort = opts.localTCPPort; + this.regctl = opts.regctl; + this.undock = opts.undock; } get toolDir(): string { return this._toolDir || Context.tmpDir(); } - async downloadStaticArchive(component: 'docker' | 'docker-rootless-extras', src: InstallSourceArchive): Promise { - const release: GitHubRelease = await Install.getRelease(src.version); - this._version = release.tag_name.replace(/^v+|v+$/g, ''); - core.debug(`docker.Install.download version: ${this._version}`); - - const downloadURL = this.downloadURL(component, this._version, src.channel); - core.info(`Downloading ${downloadURL}`); - - const downloadPath = await tc.downloadTool(downloadURL); - core.debug(`docker.Install.download downloadPath: ${downloadPath}`); - - let extractFolder; - if (os.platform() == 'win32') { - extractFolder = await tc.extractZip(downloadPath, extractFolder); - } else { - extractFolder = await tc.extractTar(downloadPath, extractFolder); - } - if (Util.isDirectory(path.join(extractFolder, component))) { - extractFolder = path.join(extractFolder, component); - } - core.debug(`docker.Install.download extractFolder: ${extractFolder}`); - return extractFolder; - } - public async download(): Promise { let extractFolder: string; let cacheKey: string; @@ -128,39 +113,9 @@ export class Install { switch (this.source.type) { case 'image': { - const tag = this.source.tag; - this._version = tag; + this._version = this.source.tag; cacheKey = `docker-image`; - - core.info(`Downloading docker cli from dockereng/cli-bin:${tag}`); - const cli = await HubRepository.build('dockereng/cli-bin'); - extractFolder = await cli.extractImage(tag); - - const moby = await HubRepository.build('moby/moby-bin'); - if (['win32', 'linux'].includes(platform)) { - core.info(`Downloading dockerd from moby/moby-bin:${tag}`); - await moby.extractImage(tag, extractFolder); - } else if (platform == 'darwin') { - // On macOS, the docker daemon binary will be downloaded inside the lima VM. - // However, we will get the exact git revision from the image config - // to get the matching systemd unit files. - core.info(`Getting git revision from moby/moby-bin:${tag}`); - - // There's no macOS image for moby/moby-bin - a linux daemon is run inside lima. - const manifest = await moby.getPlatformManifest(tag, 'linux'); - - const config = await moby.getJSONBlob(manifest.config.digest); - core.debug(`Config ${JSON.stringify(config.config)}`); - - this.gitCommit = config.config?.Labels?.['org.opencontainers.image.revision']; - if (!this.gitCommit) { - core.warning(`No git revision can be determined from the image. Will use master.`); - this.gitCommit = 'master'; - } - core.info(`Git revision is ${this.gitCommit}`); - } else { - core.warning(`dockerd not supported on ${platform}, only the Docker cli will be available`); - } + extractFolder = await this.downloadSourceImage(platform); break; } case 'archive': { @@ -170,10 +125,10 @@ export class Install { this._version = version; core.info(`Downloading Docker ${version} from ${this.source.channel} at download.docker.com`); - extractFolder = await this.downloadStaticArchive('docker', this.source); + extractFolder = await this.downloadSourceArchive('docker', this.source); if (this.rootless) { core.info(`Downloading Docker rootless extras ${version} from ${this.source.channel} at download.docker.com`); - const extrasFolder = await this.downloadStaticArchive('docker-rootless-extras', this.source); + const extrasFolder = await this.downloadSourceArchive('docker-rootless-extras', this.source); fs.readdirSync(extrasFolder).forEach(file => { const src = path.join(extrasFolder, file); const dest = path.join(extractFolder, file); @@ -191,7 +146,9 @@ export class Install { } // eslint-disable-next-line @typescript-eslint/no-unused-vars files.forEach(function (file, index) { - fs.chmodSync(path.join(extractFolder, file), '0755'); + if (!Util.isDirectory(path.join(extractFolder, file))) { + fs.chmodSync(path.join(extractFolder, file), '0755'); + } }); }); @@ -203,6 +160,72 @@ export class Install { return tooldir; } + private async downloadSourceImage(platform: string): Promise { + const dest = path.join(Context.tmpDir(), 'docker-install-image'); + const cliImage = `dockereng/cli-bin:${this._version}`; + const engineImage = `moby/moby-bin:${this._version}`; + + core.info(`Downloading Docker CLI from ${cliImage}`); + await this.undock.run({ + source: cliImage, + dist: dest + }); + + if (['win32', 'linux'].includes(platform)) { + core.info(`Downloading Docker engine from ${engineImage}`); + await this.undock.run({ + source: engineImage, + dist: dest + }); + } else if (platform == 'darwin') { + // On macOS, the docker daemon binary will be downloaded inside the lima VM. + // However, we will get the exact git revision from the image config + // to get the matching systemd unit files. There's no macOS image for + // moby/moby-bin - a linux daemon is run inside lima. + try { + const engineImageConfig = await this.imageConfig(engineImage, 'linux/arm64'); + core.debug(`docker.Install.downloadSourceImage engineImageConfig: ${JSON.stringify(engineImageConfig)}`); + this.gitCommit = engineImageConfig.config?.Labels?.['org.opencontainers.image.revision']; + if (!this.gitCommit) { + throw new Error(`No git revision can be determined from the image`); + } + } catch (e) { + core.warning(e); + this.gitCommit = 'master'; + } + + core.debug(`docker.Install.downloadSourceImage gitCommit: ${this.gitCommit}`); + } else { + core.warning(`Docker engine not supported on ${platform}, only the Docker cli will be available`); + } + + return dest; + } + + private async downloadSourceArchive(component: 'docker' | 'docker-rootless-extras', src: InstallSourceArchive): Promise { + const release: GitHubRelease = await Install.getRelease(src.version); + this._version = release.tag_name.replace(/^v+|v+$/g, ''); + core.debug(`docker.Install.downloadSourceArchive version: ${this._version}`); + + const downloadURL = this.downloadURL(component, this._version, src.channel); + core.info(`Downloading ${downloadURL}`); + + const downloadPath = await tc.downloadTool(downloadURL); + core.debug(`docker.Install.downloadSourceArchive downloadPath: ${downloadPath}`); + + let extractFolder; + if (os.platform() == 'win32') { + extractFolder = await tc.extractZip(downloadPath, extractFolder); + } else { + extractFolder = await tc.extractTar(downloadPath, extractFolder); + } + if (Util.isDirectory(path.join(extractFolder, component))) { + extractFolder = path.join(extractFolder, component); + } + core.debug(`docker.Install.downloadSourceArchive extractFolder: ${extractFolder}`); + return extractFolder; + } + public async install(): Promise { if (!this.toolDir) { throw new Error('toolDir must be set. Run download first.'); @@ -709,4 +732,20 @@ EOF`, } return res; } + + private async imageConfig(image: string, platform?: string): Promise { + const manifest = await this.regctl.manifestGet({ + image: image, + platform: platform + }); + const configDigest = manifest?.config?.digest; + if (!configDigest) { + throw new Error(`No config digest found for image ${image}`); + } + const blob = await this.regctl.blobGet({ + repository: image, + digest: configDigest + }); + return JSON.parse(blob); + } } diff --git a/src/hubRepository.ts b/src/hubRepository.ts deleted file mode 100644 index 23d1f48e..00000000 --- a/src/hubRepository.ts +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Copyright 2023 actions-toolkit authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as httpm from '@actions/http-client'; -import {Index} from './types/oci'; -import os from 'os'; -import * as core from '@actions/core'; -import {Manifest} from './types/oci/manifest'; -import * as tc from '@actions/tool-cache'; -import fs from 'fs'; -import {MEDIATYPE_IMAGE_CONFIG_V1, MEDIATYPE_IMAGE_INDEX_V1, MEDIATYPE_IMAGE_MANIFEST_V1} from './types/oci/mediatype'; -import {MEDIATYPE_IMAGE_CONFIG_V1 as DOCKER_MEDIATYPE_IMAGE_CONFIG_V1, MEDIATYPE_IMAGE_MANIFEST_LIST_V2, MEDIATYPE_IMAGE_MANIFEST_V2} from './types/docker/mediatype'; -import {DockerHub} from './dockerhub'; - -export class HubRepository { - private repo: string; - private token: string; - private static readonly http: httpm.HttpClient = new httpm.HttpClient('setup-docker-action'); - - private constructor(repository: string, token: string) { - this.repo = repository; - this.token = token; - } - - public static async build(repository: string): Promise { - const token = await this.getToken(repository); - return new HubRepository(repository, token); - } - - public async getPlatformManifest(tagOrDigest: string, os?: string): Promise { - const index = await this.getManifest(tagOrDigest); - if (index.mediaType != MEDIATYPE_IMAGE_INDEX_V1 && index.mediaType != MEDIATYPE_IMAGE_MANIFEST_LIST_V2) { - core.error(`Unsupported image media type: ${index.mediaType}`); - throw new Error(`Unsupported image media type: ${index.mediaType}`); - } - const digest = HubRepository.getPlatformManifestDigest(index, os); - return await this.getManifest(digest); - } - - // Unpacks the image layers and returns the path to the extracted image. - // Only OCI indexes/manifest list are supported for now. - public async extractImage(tag: string, destDir?: string): Promise { - const manifest = await this.getPlatformManifest(tag); - - const paths = manifest.layers.map(async layer => { - const url = this.blobUrl(layer.digest); - - return await tc.downloadTool(url, undefined, undefined, { - authorization: `Bearer ${this.token}` - }); - }); - - let files = await Promise.all(paths); - let extractFolder: string; - if (!destDir) { - extractFolder = await tc.extractTar(files[0]); - files = files.slice(1); - } else { - extractFolder = destDir; - } - - await Promise.all( - files.map(async file => { - return await tc.extractTar(file, extractFolder); - }) - ); - - fs.readdirSync(extractFolder).forEach(file => { - core.info(`extractImage(${this.repo}:${tag}) file: ${file}`); - }); - - return extractFolder; - } - - private static async getToken(repo: string): Promise { - const url = `https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull`; - - const resp = await this.http.get(url); - const body = await resp.readBody(); - const statusCode = resp.message.statusCode || 500; - if (statusCode != 200) { - throw DockerHub.parseError(resp, body); - } - - const json = JSON.parse(body); - return json.token; - } - - private blobUrl(digest: string): string { - return `https://registry-1.docker.io/v2/${this.repo}/blobs/${digest}`; - } - - public async getManifest(tagOrDigest: string): Promise { - return await this.registryGet(tagOrDigest, 'manifests', [MEDIATYPE_IMAGE_INDEX_V1, MEDIATYPE_IMAGE_MANIFEST_LIST_V2, MEDIATYPE_IMAGE_MANIFEST_V1, MEDIATYPE_IMAGE_MANIFEST_V2]); - } - - public async getJSONBlob(tagOrDigest: string): Promise { - return await this.registryGet(tagOrDigest, 'blobs', [MEDIATYPE_IMAGE_CONFIG_V1, DOCKER_MEDIATYPE_IMAGE_CONFIG_V1]); - } - - private async registryGet(tagOrDigest: string, endpoint: 'manifests' | 'blobs', accept: Array): Promise { - const url = `https://registry-1.docker.io/v2/${this.repo}/${endpoint}/${tagOrDigest}`; - - const headers = { - Authorization: `Bearer ${this.token}`, - Accept: accept.join(', ') - }; - - const resp = await HubRepository.http.get(url, headers); - const body = await resp.readBody(); - const statusCode = resp.message.statusCode || 500; - if (statusCode != 200) { - core.error(`registryGet(${this.repo}:${tagOrDigest}) failed: ${statusCode} ${body}`); - throw DockerHub.parseError(resp, body); - } - - return JSON.parse(body); - } - - private static getPlatformManifestDigest(index: Index, osOverride?: string): string { - // This doesn't handle all possible platforms normalizations, but it's good enough for now. - let pos: string = osOverride || os.platform(); - if (pos == 'win32') { - pos = 'windows'; - } - let arch = os.arch(); - if (arch == 'x64') { - arch = 'amd64'; - } - let variant = ''; - if (arch == 'arm') { - variant = 'v7'; - } - - const manifest = index.manifests.find(m => { - if (!m.platform) { - return false; - } - if (m.platform.os != pos) { - core.debug(`Skipping manifest ${m.digest} because of os: ${m.platform.os} != ${pos}`); - return false; - } - if (m.platform.architecture != arch) { - core.debug(`Skipping manifest ${m.digest} because of arch: ${m.platform.architecture} != ${arch}`); - return false; - } - if ((m.platform.variant || '') != variant) { - core.debug(`Skipping manifest ${m.digest} because of variant: ${m.platform.variant} != ${variant}`); - return false; - } - - return true; - }); - if (!manifest) { - core.error(`Cannot find manifest for ${pos}/${arch}/${variant}`); - throw new Error(`Cannot find manifest for ${pos}/${arch}/${variant}`); - } - - return manifest.digest; - } -}