Merge pull request #200 from crazy-max/docker-pull

docker: parseRepoTag and pull methods

.github/buildx-releases.json

@@ -40,6 +40,47 @@
       "https://github.com/docker/buildx/releases/download/v0.13.1/checksums.txt"
     ]
   },
+  "v0.14.0-rc1": {
+    "id": 150784571,
+    "tag_name": "v0.14.0-rc1",
+    "html_url": "https://github.com/docker/buildx/releases/tag/v0.14.0-rc1",
+    "assets": [
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.darwin-amd64",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.darwin-amd64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.darwin-amd64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.darwin-arm64",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.darwin-arm64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.darwin-arm64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-amd64",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-amd64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-amd64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-arm-v6",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-arm-v6.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-arm-v6.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-arm-v7",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-arm-v7.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-arm-v7.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-arm64",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-arm64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-arm64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-ppc64le",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-ppc64le.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-ppc64le.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-riscv64",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-riscv64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-riscv64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-s390x",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-s390x.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.linux-s390x.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.windows-amd64.exe",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.windows-amd64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.windows-amd64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.windows-arm64.exe",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.windows-arm64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/buildx-v0.14.0-rc1.windows-arm64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.14.0-rc1/checksums.txt"
+    ]
+  },
   "v0.13.1": {
     "id": 146098987,
     "tag_name": "v0.13.1",

.github/docker-releases.json

@@ -1,8 +1,14 @@
 {
   "latest": {
-    "id": 145844215,
-    "tag_name": "v26.0.0",
-    "html_url": "https://github.com/moby/moby/releases/tag/v26.0.0",
+    "id": 149921469,
+    "tag_name": "v26.0.1",
+    "html_url": "https://github.com/moby/moby/releases/tag/v26.0.1",
+    "assets": []
+  },
+  "v26.0.1": {
+    "id": 149921469,
+    "tag_name": "v26.0.1",
+    "html_url": "https://github.com/moby/moby/releases/tag/v26.0.1",
     "assets": []
   },
   "v23.0.10": {

.github/workflows/test.yml

@@ -15,6 +15,7 @@ on:
 
 env:
   NODE_VERSION: "20"
+  BUILDX_VERSION: "v0.14.0-rc1"
 
 jobs:
   test:
@@ -102,6 +103,13 @@ jobs:
         with:
           node-version: ${{ env.NODE_VERSION }}
           cache: 'yarn'
+      -
+        name: Set up Docker Buildx
+        if: startsWith(matrix.os, 'ubuntu')
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: ${{ env.BUILDX_VERSION }}
+          driver: docker
       -
         name: Install
         run: yarn install

__tests__/buildx/bake.test.itg.ts

@@ -35,14 +35,28 @@ maybe('getDefinition', () => {
     [
       'https://github.com/docker/buildx.git#v0.10.4',
       ['binaries-cross'],
-      path.join(fixturesDir, 'bake-buildx-0.10.4-binaries-cross.json')
+      path.join(fixturesDir, 'bake-buildx-0.10.4-binaries-cross.json'),
+      false,
     ],
-  ])('given %p', async (source: string, targets: string[], out: string) => {
+    // TODO: uncomment this test case when we have access to the private repo using an access token
+    // [
+    //   'https://github.com/docker/test-docker-action.git#remote-private',
+    //   ['default'],
+    //   path.join(fixturesDir, 'bake-test-docker-action-remote-private.json'),
+    //   true,
+    // ]
+  ])('given %p', async (source: string, targets: string[], out: string, auth) => {
+    const gitAuthToken = process.env.GITHUB_TOKEN || '';
+    if (auth && !gitAuthToken) {
+      console.log(`Git auth token not available, skipping test`);
+      return;
+    }
     const bake = new Bake();
     const expectedDef = <BakeDefinition>JSON.parse(fs.readFileSync(out, {encoding: 'utf-8'}).trim())
     expect(await bake.getDefinition({
       source: source,
-      targets: targets
+      targets: targets,
+      githubToken: gitAuthToken,
     })).toEqual(expectedDef);
   });
 });

__tests__/docker/docker.test.itg.ts

@@ -0,0 +1,56 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {describe, expect, test} from '@jest/globals';
+
+import {Docker} from '../../src/docker/docker';
+
+const maybe = !process.env.GITHUB_ACTIONS || (process.env.GITHUB_ACTIONS === 'true' && process.env.ImageOS && process.env.ImageOS.startsWith('ubuntu')) ? describe : describe.skip;
+
+maybe('pull', () => {
+  // prettier-ignore
+  test.each([
+    [
+      'busybox',
+      undefined,
+    ],
+    [
+      'busybox:1.36',
+      undefined,
+    ],
+    [
+      'busybox@sha256:7ae8447f3a7f5bccaa765926f25fc038e425cf1b2be6748727bbea9a13102094',
+      undefined,
+    ],
+    [
+      'doesnotexist:foo',
+      `pull access denied for doesnotexist`,
+    ],
+  ])('pulling %p', async (image: string, err: string | undefined) => {
+      try {
+        await Docker.pull(image, true);
+        if (err !== undefined) {
+          throw new Error('Expected an error to be thrown');
+        }
+      } catch (e) {
+        if (err === undefined) {
+          throw new Error(`Expected no error, but got: ${e.message}`);
+        }
+        // eslint-disable-next-line jest/no-conditional-expect
+        expect(e.message).toContain(err);
+      }
+    }, 600000);
+});

__tests__/fixtures/bake-test-docker-action-remote-private.json

@@ -0,0 +1,11 @@
+{
+  "target": {
+    "default": {
+      "context": "https://github.com/docker/test-docker-action.git#remote-private",
+      "dockerfile": "Dockerfile",
+      "tags": [
+        "foo"
+      ]
+    }
+  }
+}

dev.Dockerfile

@@ -16,7 +16,7 @@
 
 ARG NODE_VERSION=20
 ARG DOCKER_VERSION=26.0.0
-ARG BUILDX_VERSION=0.13.1
+ARG BUILDX_VERSION=0.14.0-rc1
 
 FROM node:${NODE_VERSION}-alpine AS base
 RUN apk add --no-cache cpio findutils git

package.json

@@ -58,14 +58,14 @@
     "csv-parse": "^5.5.5",
     "handlebars": "^4.7.8",
     "jwt-decode": "^4.0.0",
-    "semver": "^7.5.4",
+    "semver": "^7.6.0",
     "tmp": "^0.2.3"
   },
   "devDependencies": {
     "@types/csv-parse": "^1.2.2",
     "@types/js-yaml": "^4.0.5",
     "@types/node": "^20.5.9",
-    "@types/semver": "^7.5.1",
+    "@types/semver": "^7.5.8",
     "@types/tmp": "^0.2.3",
     "@typescript-eslint/eslint-plugin": "^6.6.0",
     "@typescript-eslint/parser": "^6.6.0",

src/buildx/bake.ts

@@ -36,6 +36,8 @@ export interface BakeCmdOpts {
   sbom?: string;
   source?: string;
   targets?: Array<string>;
+
+  githubToken?: string; // for auth with remote definitions on private repos
 }
 
 export class Bake {
@@ -48,6 +50,13 @@ export class Bake {
   public async getDefinition(cmdOpts: BakeCmdOpts, execOptions?: ExecOptions): Promise<BakeDefinition> {
     execOptions = execOptions || {ignoreReturnCode: true};
     execOptions.ignoreReturnCode = true;
+    if (cmdOpts.githubToken) {
+      execOptions.env = Object.assign({}, process.env, {
+        BUILDX_BAKE_GIT_AUTH_TOKEN: cmdOpts.githubToken
+      }) as {
+        [key: string]: string;
+      };
+    }
 
     const args = ['bake'];
 

src/buildx/inputs.ts

@@ -77,24 +77,23 @@ export class Inputs {
   }
 
   public static resolveBuildSecretString(kvp: string): string {
-    return Inputs.resolveBuildSecret(kvp, false);
+    const [key, file] = Inputs.resolveBuildSecret(kvp, false);
+    return `id=${key},src=${file}`;
   }
 
   public static resolveBuildSecretFile(kvp: string): string {
-    return Inputs.resolveBuildSecret(kvp, true);
+    const [key, file] = Inputs.resolveBuildSecret(kvp, true);
+    return `id=${key},src=${file}`;
   }
 
   public static resolveBuildSecretEnv(kvp: string): string {
     const [key, value] = parseKvp(kvp);
-
     return `id=${key},env=${value}`;
   }
 
-  public static resolveBuildSecret(kvp: string, file: boolean): string {
+  public static resolveBuildSecret(kvp: string, file: boolean): [string, string] {
     const [key, _value] = parseKvp(kvp);
-
     let value = _value;
-
     if (file) {
       if (!fs.existsSync(value)) {
         throw new Error(`secret file ${value} not found`);
@@ -103,7 +102,7 @@ export class Inputs {
     }
     const secretFile = Context.tmpName({tmpdir: Context.tmpDir()});
     fs.writeFileSync(secretFile, value);
-    return `id=${key},src=${secretFile}`;
+    return [key, secretFile];
   }
 
   public static getProvenanceInput(name: string): string {

src/docker/docker.ts

@@ -20,7 +20,10 @@ import path from 'path';
 import * as core from '@actions/core';
 import * as io from '@actions/io';
 
+import {Context} from '../context';
+import {Cache} from '../cache';
 import {Exec} from '../exec';
+import {Util} from '../util';
 
 import {ConfigFile} from '../types/docker';
 
@@ -73,4 +76,89 @@ export class Docker {
   public static async printInfo(): Promise<void> {
     await Exec.exec('docker', ['info']);
   }
+
+  public static parseRepoTag(image: string): {repository: string; tag: string} {
+    let sepPos: number;
+    const digestPos = image.indexOf('@');
+    const colonPos = image.lastIndexOf(':');
+    if (digestPos >= 0) {
+      // priority on digest
+      sepPos = digestPos;
+    } else if (colonPos >= 0) {
+      sepPos = colonPos;
+    } else {
+      return {
+        repository: image,
+        tag: 'latest'
+      };
+    }
+    const tag = image.slice(sepPos + 1);
+    if (tag.indexOf('/') === -1) {
+      return {
+        repository: image.slice(0, sepPos),
+        tag: tag
+      };
+    }
+    return {
+      repository: image,
+      tag: 'latest'
+    };
+  }
+
+  public static async pull(image: string, cache?: boolean): Promise<void> {
+    const parsedImage = Docker.parseRepoTag(image);
+    const repoSanitized = parsedImage.repository.replace(/[^a-zA-Z0-9.]+/g, '--');
+    const tagSanitized = parsedImage.tag.replace(/[^a-zA-Z0-9.]+/g, '--');
+
+    const imageCache = new Cache({
+      htcName: repoSanitized,
+      htcVersion: tagSanitized,
+      baseCacheDir: path.join(Docker.configDir, '.cache', 'images', repoSanitized),
+      cacheFile: 'image.tar'
+    });
+
+    let cacheFoundPath: string | undefined;
+    if (cache) {
+      cacheFoundPath = await imageCache.find();
+      if (cacheFoundPath) {
+        core.info(`Image found from cache in ${cacheFoundPath}`);
+        await Exec.getExecOutput(`docker`, ['load', '-i', cacheFoundPath], {
+          ignoreReturnCode: true
+        }).then(res => {
+          if (res.stderr.length > 0 && res.exitCode != 0) {
+            core.warning(`Failed to load image from cache: ${res.stderr.match(/(.*)\s*$/)?.[0]?.trim() ?? 'unknown error'}`);
+          }
+        });
+      }
+    }
+
+    let pulled = true;
+    await Exec.getExecOutput(`docker`, ['pull', image], {
+      ignoreReturnCode: true
+    }).then(res => {
+      pulled = false;
+      if (res.stderr.length > 0 && res.exitCode != 0) {
+        const err = res.stderr.match(/(.*)\s*$/)?.[0]?.trim() ?? 'unknown error';
+        if (cacheFoundPath) {
+          core.warning(`Failed to pull image, using one from cache: ${err}`);
+        } else {
+          throw new Error(err);
+        }
+      }
+    });
+
+    if (cache && pulled) {
+      const imageTarPath = path.join(Context.tmpDir(), `${Util.hash(image)}.tar`);
+      await Exec.getExecOutput(`docker`, ['save', '-o', imageTarPath, image], {
+        ignoreReturnCode: true
+      }).then(async res => {
+        if (res.stderr.length > 0 && res.exitCode != 0) {
+          core.warning(`Failed to save image: ${res.stderr.match(/(.*)\s*$/)?.[0]?.trim() ?? 'unknown error'}`);
+        } else {
+          const cachePath = await imageCache.save(imageTarPath);
+          core.info(`Image cached to ${cachePath}`);
+        }
+      });
+    }
+  }
 }

yarn.lock

@@ -1061,7 +1061,7 @@ __metadata:
     "@types/csv-parse": ^1.2.2
     "@types/js-yaml": ^4.0.5
     "@types/node": ^20.5.9
-    "@types/semver": ^7.5.1
+    "@types/semver": ^7.5.8
     "@types/tmp": ^0.2.3
     "@typescript-eslint/eslint-plugin": ^6.6.0
     "@typescript-eslint/parser": ^6.6.0
@@ -1078,7 +1078,7 @@ __metadata:
     jwt-decode: ^4.0.0
     prettier: ^3.0.3
     rimraf: ^5.0.1
-    semver: ^7.5.4
+    semver: ^7.6.0
     tmp: ^0.2.3
     ts-jest: ^29.1.1
     ts-node: ^10.9.1
@@ -2013,10 +2013,10 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/semver@npm:^7.5.1":
-  version: 7.5.1
-  resolution: "@types/semver@npm:7.5.1"
-  checksum: 2fffe938c7ac168711f245a16e1856a3578d77161ca17e29a05c3e02c7be3e9c5beefa29a3350f6c1bd982fb70aa28cc52e4845eb7d36246bcdc0377170d584d
+"@types/semver@npm:^7.5.8":
+  version: 7.5.8
+  resolution: "@types/semver@npm:7.5.8"
+  checksum: ea6f5276f5b84c55921785a3a27a3cd37afee0111dfe2bcb3e03c31819c197c782598f17f0b150a69d453c9584cd14c4c4d7b9a55d2c5e6cacd4d66fdb3b3663
   languageName: node
   linkType: hard
 
@@ -6502,6 +6502,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"semver@npm:^7.6.0":
+  version: 7.6.0
+  resolution: "semver@npm:7.6.0"
+  dependencies:
+    lru-cache: ^6.0.0
+  bin:
+    semver: bin/semver.js
+  checksum: 7427f05b70786c696640edc29fdd4bc33b2acf3bbe1740b955029044f80575fc664e1a512e4113c3af21e767154a94b4aa214bf6cd6e42a1f6dba5914e0b208c
+  languageName: node
+  linkType: hard
+
 "set-blocking@npm:^2.0.0":
   version: 2.0.0
   resolution: "set-blocking@npm:2.0.0"
@@ -7169,11 +7180,11 @@ __metadata:
   linkType: hard
 
 "undici@npm:^5.25.4":
-  version: 5.28.3
-  resolution: "undici@npm:5.28.3"
+  version: 5.28.4
+  resolution: "undici@npm:5.28.4"
   dependencies:
     "@fastify/busboy": ^2.0.0
-  checksum: fa1e65aff896c5e2ee23637b632e306f9e3a2b32a3dc0b23ea71e5555ad350bcc25713aea894b3dccc0b7dc2c5e92a5a58435ebc2033b731a5524506f573dfd2
+  checksum: a8193132d84540e4dc1895ecc8dbaa176e8a49d26084d6fbe48a292e28397cd19ec5d13bc13e604484e76f94f6e334b2bdc740d5f06a6e50c44072818d0c19f9
   languageName: node
   linkType: hard