Merge pull request #42 from crazy-max/buildkit-fix-version

buildkit: use node info to retrieve version

__tests__/buildkit/buildkit.test.ts

@@ -15,7 +15,6 @@
  */
 
 import {beforeEach, describe, expect, it, jest, test} from '@jest/globals';
-import * as semver from 'semver';
 
 import {BuildKit} from '../../src/buildkit/buildkit';
 import {Builder} from '../../src/buildx/builder';
@@ -48,11 +47,15 @@ jest.spyOn(Builder.prototype, 'inspect').mockImplementation(async (): Promise<Bu
 
 describe('getVersion', () => {
   it('valid', async () => {
+    const builder = new Builder({
+      context: new Context()
+    });
+    const builderInfo = await builder.inspect('builder2');
     const buildkit = new BuildKit({
       context: new Context()
     });
-    const version = await buildkit.getVersion('builder2');
-    expect(semver.valid(version)).not.toBeNull();
+    const version = await buildkit.getVersion(builderInfo.nodes[0]);
+    expect(version).toBe('v0.11.0');
   });
 });
 

__tests__/buildx/install.test.ts

@@ -14,31 +14,36 @@
  * limitations under the License.
  */
 
-import {describe, expect, it, jest, test, beforeEach} from '@jest/globals';
+import {describe, expect, it, jest, test, beforeEach, afterEach} from '@jest/globals';
 import * as fs from 'fs';
 import * as path from 'path';
+import * as rimraf from 'rimraf';
 import osm = require('os');
 
 import {Install} from '../../src/buildx/install';
 
+// prettier-ignore
+const tmpDir = path.join(process.env.TEMP || '/tmp', 'buildx-jest').split(path.sep).join(path.posix.sep);
+
 beforeEach(() => {
   jest.clearAllMocks();
 });
 
-describe('install', () => {
-  // prettier-ignore
-  const tmpDir = path.join(process.env.TEMP || '/tmp', 'buildx-install-jest').split(path.sep).join(path.posix.sep);
+afterEach(function () {
+  rimraf.sync(tmpDir);
+});
 
+describe('download', () => {
   // prettier-ignore
   test.each([
-    ['v0.4.1', false],
+    ['v0.9.1', false],
     ['latest', false],
-    ['v0.4.1', true],
+    ['v0.9.1', true],
     ['latest', true]
   ])(
   'acquires %p of buildx (standalone: %p)', async (version, standalone) => {
       const install = new Install({standalone: standalone});
-      const buildxBin = await install.install(version, tmpDir);
+      const buildxBin = await install.download(version, tmpDir);
       expect(fs.existsSync(buildxBin)).toBe(true);
     },
     100000
@@ -60,7 +65,7 @@ describe('install', () => {
       jest.spyOn(osm, 'platform').mockImplementation(() => os);
       jest.spyOn(osm, 'arch').mockImplementation(() => arch);
       const install = new Install();
-      const buildxBin = await install.install('latest', tmpDir);
+      const buildxBin = await install.download('latest', tmpDir);
       expect(fs.existsSync(buildxBin)).toBe(true);
     },
     100000
@@ -73,6 +78,22 @@ describe('install', () => {
   });
 });
 
+describe('build', () => {
+  // eslint-disable-next-line jest/no-disabled-tests
+  it.skip('builds refs/pull/648/head', async () => {
+    const install = new Install();
+    const buildxBin = await install.build('https://github.com/docker/buildx.git#refs/pull/648/head', tmpDir);
+    expect(fs.existsSync(buildxBin)).toBe(true);
+  }, 100000);
+
+  // eslint-disable-next-line jest/no-disabled-tests
+  it.skip('builds 67bd6f4dc82a9cd96f34133dab3f6f7af803bb14', async () => {
+    const install = new Install();
+    const buildxBin = await install.build('https://github.com/docker/buildx.git#67bd6f4dc82a9cd96f34133dab3f6f7af803bb14', tmpDir);
+    expect(fs.existsSync(buildxBin)).toBe(true);
+  }, 100000);
+});
+
 describe('getRelease', () => {
   it('returns latest buildx GitHub release', async () => {
     const release = await Install.getRelease('latest');

__tests__/docker.test.ts

@@ -48,9 +48,9 @@ describe('configDir', () => {
 });
 
 describe('isAvailable', () => {
-  it('cli', () => {
+  it('cli', async () => {
     const execSpy = jest.spyOn(exec, 'getExecOutput');
-    Docker.isAvailable;
+    Docker.getInstance().available;
     // eslint-disable-next-line jest/no-standalone-expect
     expect(execSpy).toHaveBeenCalledWith(`docker`, undefined, {
       silent: true,

__tests__/util.test.ts

@@ -69,16 +69,28 @@ describe('getInputList', () => {
 
   it('multiline and ignoring comma correctly', async () => {
     setInput('cache-from', 'user/app:cache\ntype=local,src=path/to/dir');
-    const res = Util.getInputList('cache-from', true);
+    const res = Util.getInputList('cache-from', {ignoreComma: true});
     expect(res).toEqual(['user/app:cache', 'type=local,src=path/to/dir']);
   });
 
   it('different new lines and ignoring comma correctly', async () => {
     setInput('cache-from', 'user/app:cache\r\ntype=local,src=path/to/dir');
-    const res = Util.getInputList('cache-from', true);
+    const res = Util.getInputList('cache-from', {ignoreComma: true});
     expect(res).toEqual(['user/app:cache', 'type=local,src=path/to/dir']);
   });
 
+  it('do not escape surrounding quotes', async () => {
+    setInput('driver-opts', `"env.no_proxy=localhost,127.0.0.1,.mydomain"`);
+    const res = Util.getInputList('driver-opts', {ignoreComma: true, quote: false});
+    expect(res).toEqual(['"env.no_proxy=localhost,127.0.0.1,.mydomain"']);
+  });
+
+  it('escape surrounding quotes', async () => {
+    setInput('platforms', 'linux/amd64\n"linux/arm64,linux/arm/v7"');
+    const res = Util.getInputList('platforms');
+    expect(res).toEqual(['linux/amd64', 'linux/arm64', 'linux/arm/v7']);
+  });
+
   it('multiline values', async () => {
     setInput(
       'secrets',
@@ -88,7 +100,7 @@ bbbbbbb
 ccccccccc"
 FOO=bar`
     );
-    const res = Util.getInputList('secrets', true);
+    const res = Util.getInputList('secrets', {ignoreComma: true});
     expect(res).toEqual([
       'GIT_AUTH_TOKEN=abcdefgh,ijklmno=0123456789',
       `MYSECRET=aaaaaaaa
@@ -111,7 +123,7 @@ FOO=bar
 bbbb
 ccc"`
     );
-    const res = Util.getInputList('secrets', true);
+    const res = Util.getInputList('secrets', {ignoreComma: true});
     expect(res).toEqual([
       'GIT_AUTH_TOKEN=abcdefgh,ijklmno=0123456789',
       `MYSECRET=aaaaaaaa
@@ -134,7 +146,7 @@ bbbbbbb
 ccccccccc
 FOO=bar`
     );
-    const res = Util.getInputList('secrets', true);
+    const res = Util.getInputList('secrets', {ignoreComma: true});
     expect(res).toEqual(['GIT_AUTH_TOKEN=abcdefgh,ijklmno=0123456789', 'MYSECRET=aaaaaaaa', 'bbbbbbb', 'ccccccccc', 'FOO=bar']);
   });
 
@@ -145,7 +157,7 @@ FOO=bar`
       `"GPG_KEY=${pgp}"
 FOO=bar`
     );
-    const res = Util.getInputList('secrets', true);
+    const res = Util.getInputList('secrets', {ignoreComma: true});
     expect(res).toEqual([`GPG_KEY=${pgp}`, 'FOO=bar']);
   });
 
@@ -158,7 +170,7 @@ bbbb""bbb
 ccccccccc"
 FOO=bar`
     );
-    const res = Util.getInputList('secrets', true);
+    const res = Util.getInputList('secrets', {ignoreComma: true});
     expect(res).toEqual([
       'GIT_AUTH_TOKEN=abcdefgh,ijklmno=0123456789',
       `MYSECRET=aaaaaaaa

package.json

@@ -1,5 +1,6 @@
 {
   "name": "@docker/actions-toolkit",
+  "version": "0.0.0+unknown",
   "description": "Toolkit for Docker (GitHub) Actions",
   "scripts": {
     "build": "tsc",

src/buildkit/buildkit.ts

@@ -23,7 +23,7 @@ import {Buildx} from '../buildx/buildx';
 import {Builder} from '../buildx/builder';
 import {Config} from './config';
 
-import {BuilderInfo} from '../types/builder';
+import {BuilderInfo, NodeInfo} from '../types/builder';
 
 export interface BuildKitOpts {
   context: Context;
@@ -33,7 +33,6 @@ export interface BuildKitOpts {
 export class BuildKit {
   private readonly context: Context;
   private readonly buildx: Buildx;
-  private containerNamePrefix = 'buildx_buildkit_';
 
   public readonly config: Config;
 
@@ -47,14 +46,7 @@ export class BuildKit {
       });
   }
 
-  public async getVersion(builderName: string): Promise<string | undefined> {
-    const builderInfo = await this.getBuilderInfo(builderName);
-    if (builderInfo.nodes.length == 0) {
-      // a builder always have on node, should not happen.
-      return undefined;
-    }
-    // TODO: get version for all nodes
-    const node = builderInfo.nodes[0];
+  public async getVersion(node: NodeInfo): Promise<string | undefined> {
     if (!node.buildkitVersion && node.name) {
       try {
         return await this.getVersionWithinImage(node.name);
@@ -67,7 +59,7 @@ export class BuildKit {
 
   private async getVersionWithinImage(nodeName: string): Promise<string> {
     return exec
-      .getExecOutput(`docker`, ['inspect', '--format', '{{.Config.Image}}', `${this.containerNamePrefix}${nodeName}`], {
+      .getExecOutput(`docker`, ['inspect', '--format', '{{.Config.Image}}', `${Buildx.containerNamePrefix}${nodeName}`], {
         ignoreReturnCode: true,
         silent: true
       })
@@ -93,8 +85,13 @@ export class BuildKit {
       });
   }
 
-  public async versionSatisfies(builderName: string, range: string): Promise<boolean> {
-    const builderInfo = await this.getBuilderInfo(builderName);
+  public async versionSatisfies(builderName: string, range: string, builderInfo?: BuilderInfo): Promise<boolean> {
+    if (!builderInfo) {
+      builderInfo = await new Builder({
+        context: this.context,
+        buildx: this.buildx
+      }).inspect(builderName);
+    }
     for (const node of builderInfo.nodes) {
       let bkversion = node.buildkitVersion;
       if (!bkversion) {
@@ -114,12 +111,4 @@ export class BuildKit {
     }
     return true;
   }
-
-  private async getBuilderInfo(name: string): Promise<BuilderInfo> {
-    const builder = new Builder({
-      context: this.context,
-      buildx: this.buildx
-    });
-    return builder.inspect(name);
-  }
 }

src/buildx/buildx.ts

@@ -16,13 +16,13 @@
 
 import fs from 'fs';
 import path from 'path';
+import * as core from '@actions/core';
 import * as exec from '@actions/exec';
 import * as semver from 'semver';
 
 import {Docker} from '../docker';
 import {Context} from '../context';
 import {Inputs} from './inputs';
-import {Install} from './install';
 
 import {Cert} from '../types/buildx';
 
@@ -36,14 +36,13 @@ export class Buildx {
   private _version: string | undefined;
 
   public readonly inputs: Inputs;
-  public readonly install: Install;
   public readonly standalone: boolean;
+  public static readonly containerNamePrefix = 'buildx_buildkit_';
 
   constructor(opts: BuildxOpts) {
     this.context = opts.context;
+    this.standalone = opts?.standalone ?? !Docker.getInstance().available;
     this.inputs = new Inputs(this.context);
-    this.install = new Install({standalone: opts.standalone});
-    this.standalone = opts?.standalone ?? !Docker.isAvailable;
   }
 
   static get configDir(): string {
@@ -125,9 +124,12 @@ export class Buildx {
   public async versionSatisfies(range: string, version?: string): Promise<boolean> {
     const ver = version ?? (await this.version);
     if (!ver) {
+      core.debug(`Buildx.versionSatisfies false: undefined version`);
       return false;
     }
-    return semver.satisfies(ver, range) || /^[0-9a-f]{7}$/.exec(ver) !== null;
+    const res = semver.satisfies(ver, range) || /^[0-9a-f]{7}$/.exec(ver) !== null;
+    core.debug(`Buildx.versionSatisfies ${ver} statisfies ${range}: ${res}`);
+    return res;
   }
 
   public static resolveCertsDriverOpts(driver: string, endpoint: string, cert: Cert): Array<string> {

src/buildx/install.ts

@@ -18,27 +18,37 @@ import fs from 'fs';
 import os from 'os';
 import path from 'path';
 import * as core from '@actions/core';
+import * as exec from '@actions/exec';
 import * as httpm from '@actions/http-client';
 import * as tc from '@actions/tool-cache';
 import * as semver from 'semver';
 import * as util from 'util';
 
+import {Buildx} from './buildx';
+import {Context} from '../context';
+import {Docker} from '../docker';
+import {Git} from '../git';
+
 import {GitHubRelease} from '../types/github';
 
 export interface InstallOpts {
+  context?: Context;
   standalone?: boolean;
 }
 
 export class Install {
-  private readonly opts: InstallOpts;
+  private readonly context: Context;
+  private readonly standalone: boolean;
 
   constructor(opts?: InstallOpts) {
-    this.opts = opts || {};
+    this.context = opts?.context || new Context();
+    this.standalone = opts?.standalone ?? !Docker.getInstance().available;
   }
 
-  public async install(version: string, dest: string): Promise<string> {
+  public async download(version: string, dest?: string): Promise<string> {
     const release: GitHubRelease = await Install.getRelease(version);
     const fversion = release.tag_name.replace(/^v+|v+$/g, '');
+
     let toolPath: string;
     toolPath = tc.find('buildx', fversion, this.platform());
     if (!toolPath) {
@@ -46,14 +56,87 @@ export class Install {
       if (!semver.valid(c)) {
         throw new Error(`Invalid Buildx version "${fversion}".`);
       }
-      toolPath = await this.download(fversion);
+      toolPath = await this.fetchBinary(fversion);
     }
-    if (this.opts.standalone) {
+
+    dest = dest || (this.standalone ? this.context.tmpDir() : Docker.configDir);
+    if (this.standalone) {
       return this.setStandalone(toolPath, dest);
     }
     return this.setPlugin(toolPath, dest);
   }
 
+  public async build(gitContext: string, dest?: string): Promise<string> {
+    // eslint-disable-next-line prefer-const
+    let [repo, ref] = gitContext.split('#');
+    if (ref.length == 0) {
+      ref = 'master';
+    }
+
+    let vspec: string;
+    // TODO: include full ref as fingerprint. Use commit sha as best-effort in the meantime.
+    if (ref.match(/^[0-9a-fA-F]{40}$/)) {
+      vspec = ref;
+    } else {
+      vspec = await Git.getRemoteSha(repo, ref);
+    }
+    core.debug(`Install.build: tool version spec ${vspec}`);
+
+    let toolPath: string;
+    toolPath = tc.find('buildx', vspec);
+    if (!toolPath) {
+      const outputDir = path.join(this.context.tmpDir(), 'build-cache').split(path.sep).join(path.posix.sep);
+      const buildCmd = await this.buildCommand(gitContext, outputDir);
+      toolPath = await exec
+        .getExecOutput(buildCmd.command, buildCmd.args, {
+          ignoreReturnCode: true
+        })
+        .then(res => {
+          if (res.stderr.length > 0 && res.exitCode != 0) {
+            core.warning(res.stderr.trim());
+          }
+          return tc.cacheFile(`${outputDir}/buildx`, os.platform() == 'win32' ? 'docker-buildx.exe' : 'docker-buildx', 'buildx', vspec);
+        });
+    }
+
+    dest = dest || Docker.configDir;
+    if (this.standalone) {
+      return this.setStandalone(toolPath, dest);
+    }
+    return this.setPlugin(toolPath, dest);
+  }
+
+  private async buildCommand(gitContext: string, outputDir: string): Promise<{args: Array<string>; command: string}> {
+    const buildxStandaloneFound = await new Buildx({context: this.context, standalone: true}).isAvailable();
+    const buildxPluginFound = await new Buildx({context: this.context, standalone: false}).isAvailable();
+
+    let buildStandalone = false;
+    if (this.standalone && buildxStandaloneFound) {
+      core.debug(`Install.buildCommand: Buildx standalone found, build with it`);
+      buildStandalone = true;
+    } else if (!this.standalone && buildxPluginFound) {
+      core.debug(`Install.buildCommand: Buildx plugin found, build with it`);
+      buildStandalone = false;
+    } else if (buildxStandaloneFound) {
+      core.debug(`Install.buildCommand: Buildx plugin not found, but standalone found so trying to build with it`);
+      buildStandalone = true;
+    } else if (buildxPluginFound) {
+      core.debug(`Install.buildCommand: Buildx standalone not found, but plugin found so trying to build with it`);
+      buildStandalone = false;
+    } else {
+      throw new Error(`Neither buildx standalone or plugin have been found to build from ref ${gitContext}`);
+    }
+
+    //prettier-ignore
+    return new Buildx({context: this.context, standalone: buildStandalone}).getCommand([
+      'build',
+      '--target', 'binaries',
+      '--build-arg', 'BUILDKIT_CONTEXT_KEEP_GIT_DIR=1',
+      '--output', `type=local,dest=${outputDir}`,
+      gitContext
+    ]);
+  }
+
   private async setStandalone(toolPath: string, dest: string): Promise<string> {
     const toolBinPath = path.join(toolPath, os.platform() == 'win32' ? 'docker-buildx.exe' : 'docker-buildx');
     const binDir = path.join(dest, 'bin');
@@ -81,7 +164,7 @@ export class Install {
     return pluginPath;
   }
 
-  private async download(version: string): Promise<string> {
+  private async fetchBinary(version: string): Promise<string> {
     const targetFile: string = os.platform() == 'win32' ? 'docker-buildx.exe' : 'docker-buildx';
     const downloadURL = util.format('https://github.com/docker/buildx/releases/download/v%s/%s', version, this.filename(version));
     const downloadPath = await tc.downloadTool(downloadURL);

src/docker.ts

@@ -16,37 +16,53 @@
 
 import os from 'os';
 import path from 'path';
+import * as core from '@actions/core';
 import * as exec from '@actions/exec';
 
 export class Docker {
+  private static instance?: Docker;
+  static getInstance = (): Docker => (Docker.instance = Docker.instance ?? new Docker());
+
+  private _available: boolean | undefined;
+
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  private constructor() {}
+
   static get configDir(): string {
     return process.env.DOCKER_CONFIG || path.join(os.homedir(), '.docker');
   }
 
-  static get isAvailable(): boolean {
-    let dockerAvailable = false;
-    exec
-      .getExecOutput('docker', undefined, {
-        ignoreReturnCode: true,
-        silent: true
-      })
-      .then(res => {
-        if (res.stderr.length > 0 && res.exitCode != 0) {
-          dockerAvailable = false;
-        } else {
-          dockerAvailable = res.exitCode == 0;
-        }
-      })
-      // eslint-disable-next-line @typescript-eslint/no-unused-vars
-      .catch(error => {
-        dockerAvailable = false;
-      });
-    return dockerAvailable;
+  get available() {
+    return (async () => {
+      if (!this._available) {
+        this._available = await exec
+          .getExecOutput('docker', undefined, {
+            ignoreReturnCode: true,
+            silent: true
+          })
+          .then(res => {
+            if (res.stderr.length > 0 && res.exitCode != 0) {
+              core.debug(`Docker.isAvailable error: ${res.stderr}`);
+              return false;
+            } else {
+              core.debug(`Docker.isAvailable ok`);
+              return res.exitCode == 0;
+            }
+          })
+          // eslint-disable-next-line @typescript-eslint/no-unused-vars
+          .catch(error => {
+            core.debug(`Docker.isAvailable failed: ${error}`);
+            return false;
+          });
+      }
+      return this._available;
+    })();
   }
 
-  public static async printVersion(standalone?: boolean) {
-    const noDocker = standalone ?? !Docker.isAvailable;
+  public static async printVersion(standalone?: boolean): Promise<void> {
+    const noDocker = standalone ?? !Docker.getInstance().available;
     if (noDocker) {
+      core.debug('Docker.printVersion: Docker is not available, skipping.');
       return;
     }
     await exec.exec('docker', ['version'], {
@@ -54,9 +70,10 @@ export class Docker {
     });
   }
 
-  public static async printInfo(standalone?: boolean) {
-    const noDocker = standalone ?? !Docker.isAvailable;
+  public static async printInfo(standalone?: boolean): Promise<void> {
+    const noDocker = standalone ?? !Docker.getInstance().available;
     if (noDocker) {
+      core.debug('Docker.printInfo: Docker is not available, skipping.');
       return;
     }
     await exec.exec('docker', ['info'], {

src/toolkit.ts

@@ -16,8 +16,11 @@
 
 import {Context} from './context';
 import {Buildx} from './buildx/buildx';
+import {Install} from './buildx/install';
+import {Builder} from './buildx/builder';
 import {BuildKit} from './buildkit/buildkit';
 import {GitHub} from './github';
+import {Docker} from './docker';
 
 export interface ToolkitOpts {
   /**
@@ -30,13 +33,19 @@ export interface ToolkitOpts {
 export class Toolkit {
   public context: Context;
   public github: GitHub;
+  public docker: Docker;
   public buildx: Buildx;
+  public buildxInstall: Install;
+  public builder: Builder;
   public buildkit: BuildKit;
 
   constructor(opts: ToolkitOpts = {}) {
     this.context = new Context();
     this.github = new GitHub({token: opts.githubToken});
-    this.buildx = new Buildx({context: this.context});
+    this.docker = Docker.getInstance();
+    this.buildx = new Buildx({context: this.context, standalone: !this.docker.available});
+    this.buildxInstall = new Install({context: this.context, standalone: this.buildx.standalone});
+    this.builder = new Builder({context: this.context, buildx: this.buildx});
     this.buildkit = new BuildKit({context: this.context, buildx: this.buildx});
   }
 }

src/types/builder.ts

@@ -30,3 +30,11 @@ export interface NodeInfo {
   buildkitVersion?: string;
   platforms?: string;
 }
+
+export interface Node {
+  name?: string;
+  endpoint?: string;
+  'driver-opts'?: Array<string>;
+  'buildkitd-flags'?: string;
+  platforms?: string;
+}

src/util.ts

@@ -17,8 +17,13 @@
 import * as core from '@actions/core';
 import {parse} from 'csv-parse/sync';
 
+export interface InputListOpts {
+  ignoreComma?: boolean;
+  quote?: string | boolean | Buffer | null;
+}
+
 export class Util {
-  public static getInputList(name: string, ignoreComma?: boolean): string[] {
+  public static getInputList(name: string, opts?: InputListOpts): string[] {
     const res: Array<string> = [];
 
     const items = core.getInput(name);
@@ -31,18 +36,22 @@ export class Util {
       relaxQuotes: true,
       comment: '#',
       relaxColumnCount: true,
-      skipEmptyLines: true
+      skipEmptyLines: true,
+      quote: opts?.quote
     });
 
     for (const record of records as Array<string[]>) {
       if (record.length == 1) {
-        res.push(record[0]);
-        continue;
-      } else if (!ignoreComma) {
+        if (opts?.ignoreComma) {
+          res.push(record[0]);
+        } else {
+          res.push(...record[0].split(','));
+        }
+      } else if (!opts?.ignoreComma) {
         res.push(...record);
-        continue;
+      } else {
+        res.push(record.join(','));
       }
-      res.push(record.join(','));
     }
 
     return res.filter(item => item).map(pat => pat.trim());