Merge pull request #652 from crazy-max/buildx-history-cmd

history: export build using history command support

__tests__/buildx/history.test.itg.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import {afterEach, beforeEach, describe, expect, it, jest, test} from '@jest/globals';
+import {describe, expect, it, test} from '@jest/globals';
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
@@ -30,7 +30,49 @@ const tmpDir = fs.mkdtempSync(path.join(process.env.TEMP || os.tmpdir(), 'buildx
 
 const maybe = !process.env.GITHUB_ACTIONS || (process.env.GITHUB_ACTIONS === 'true' && process.env.ImageOS && process.env.ImageOS.startsWith('ubuntu')) ? describe : describe.skip;
 
-maybe('exportBuild', () => {
+maybe('inspect', () => {
+  it('build', async () => {
+    const buildx = new Buildx();
+    const build = new Build({buildx: buildx});
+
+    fs.mkdirSync(tmpDir, {recursive: true});
+    await expect(
+      (async () => {
+        // prettier-ignore
+        const buildCmd = await buildx.getCommand([
+          '--builder', process.env.CTN_BUILDER_NAME ?? 'default',
+          'build', '-f', path.join(fixturesDir, 'hello.Dockerfile'),
+          '--metadata-file', build.getMetadataFilePath(),
+          fixturesDir
+        ]);
+        await Exec.exec(buildCmd.command, buildCmd.args);
+      })()
+    ).resolves.not.toThrow();
+
+    const metadata = build.resolveMetadata();
+    expect(metadata).toBeDefined();
+    const buildRef = build.resolveRef(metadata);
+    if (!buildRef) {
+      throw new Error('buildRef is undefined');
+    }
+    const [builderName, nodeName, ref] = buildRef.split('/');
+    expect(builderName).toBeDefined();
+    expect(nodeName).toBeDefined();
+    expect(ref).toBeDefined();
+
+    const history = new History({buildx: buildx});
+    const res = await history.inspect({
+      ref: ref,
+      builder: builderName
+    });
+
+    expect(res).toBeDefined();
+    expect(res?.Name).toBeDefined();
+    expect(res?.Ref).toBeDefined();
+  });
+});
+
+maybe('export', () => {
   // prettier-ignore
   test.each([
     [
@@ -50,7 +92,7 @@ maybe('exportBuild', () => {
         fixturesDir
       ],
     ]
-  ])('export build %p', async (_, bargs) => {
+  ])('export with build %p', async (_, bargs) => {
     const buildx = new Buildx();
     const build = new Build({buildx: buildx});
 
@@ -110,7 +152,7 @@ maybe('exportBuild', () => {
         'hello-matrix'
       ],
     ]
-  ])('export bake build %p', async (_, bargs) => {
+  ])('export with bake %p', async (_, bargs) => {
     const buildx = new Buildx();
     const bake = new Bake({buildx: buildx});
 
@@ -145,22 +187,8 @@ maybe('exportBuild', () => {
     expect(fs.existsSync(exportRes?.dockerbuildFilename)).toBe(true);
     expect(exportRes?.summaries).toBeDefined();
   });
-});
 
-maybe('exportBuild custom image', () => {
-  const originalEnv = process.env;
-  beforeEach(() => {
-    jest.resetModules();
-    process.env = {
-      ...originalEnv,
-      DOCKER_BUILD_EXPORT_BUILD_IMAGE: 'docker.io/dockereng/export-build:0.2.2'
-    };
-  });
-  afterEach(() => {
-    process.env = originalEnv;
-  });
-
-  it('with custom image', async () => {
+  it('export using container', async () => {
     const buildx = new Buildx();
     const build = new Build({buildx: buildx});
 
@@ -185,7 +213,8 @@ maybe('exportBuild custom image', () => {
 
     const history = new History({buildx: buildx});
     const exportRes = await history.export({
-      refs: [buildRef ?? '']
+      refs: [buildRef ?? ''],
+      useContainer: true
     });
 
     expect(exportRes).toBeDefined();

__tests__/util.test.ts

@@ -469,6 +469,36 @@ describe('isPathRelativeTo', () => {
   });
 });
 
+describe('formatDuration', () => {
+  it('formats 0 nanoseconds as "0s"', () => {
+    expect(Util.formatDuration(0)).toBe('0s');
+  });
+  it('formats only seconds', () => {
+    expect(Util.formatDuration(5e9)).toBe('5s');
+    expect(Util.formatDuration(59e9)).toBe('59s');
+  });
+  it('formats minutes and seconds', () => {
+    expect(Util.formatDuration(65e9)).toBe('1m5s');
+    expect(Util.formatDuration(600e9)).toBe('10m');
+  });
+  it('formats hours, minutes, and seconds', () => {
+    expect(Util.formatDuration(3661e9)).toBe('1h1m1s');
+    expect(Util.formatDuration(7322e9)).toBe('2h2m2s');
+  });
+  it('formats hours only', () => {
+    expect(Util.formatDuration(3 * 3600e9)).toBe('3h');
+  });
+  it('formats hours and minutes', () => {
+    expect(Util.formatDuration(3900e9)).toBe('1h5m');
+  });
+  it('formats minutes only', () => {
+    expect(Util.formatDuration(120e9)).toBe('2m');
+  });
+  it('rounds down partial seconds', () => {
+    expect(Util.formatDuration(1799999999)).toBe('1s');
+  });
+});
+
 // See: https://github.com/actions/toolkit/blob/a1b068ec31a042ff1e10a522d8fdf0b8869d53ca/packages/core/src/core.ts#L89
 function getInputName(name: string): string {
   return `INPUT_${name.replace(/ /g, '_').toUpperCase()}`;

src/buildx/history.ts

@@ -28,7 +28,7 @@ import {Exec} from '../exec';
 import {GitHub} from '../github';
 import {Util} from '../util';
 
-import {ExportRecordOpts, ExportRecordResponse, Summaries} from '../types/buildx/history';
+import {ExportOpts, ExportResponse, InspectOpts, InspectResponse, Summaries} from '../types/buildx/history';
 
 export interface HistoryOpts {
   buildx?: Buildx;
@@ -37,27 +37,43 @@ export interface HistoryOpts {
 export class History {
   private readonly buildx: Buildx;
 
-  private static readonly EXPORT_BUILD_IMAGE_DEFAULT: string = 'docker.io/dockereng/export-build:latest';
-  private static readonly EXPORT_BUILD_IMAGE_ENV: string = 'DOCKER_BUILD_EXPORT_BUILD_IMAGE';
-
   constructor(opts?: HistoryOpts) {
     this.buildx = opts?.buildx || new Buildx();
   }
 
-  public async export(opts: ExportRecordOpts): Promise<ExportRecordResponse> {
-    if (os.platform() === 'win32') {
-      throw new Error('Exporting a build record is currently not supported on Windows');
-    }
-    if (!(await Docker.isAvailable())) {
-      throw new Error('Docker is required to export a build record');
-    }
-    if (!(await Docker.isDaemonRunning())) {
-      throw new Error('Docker daemon needs to be running to export a build record');
-    }
-    if (!(await this.buildx.versionSatisfies('>=0.13.0'))) {
-      throw new Error('Buildx >= 0.13.0 is required to export a build record');
-    }
+  public async getCommand(args: Array<string>) {
+    return await this.buildx.getCommand(['history', ...args]);
+  }
 
+  public async getInspectCommand(args: Array<string>) {
+    return await this.getCommand(['inspect', ...args]);
+  }
+
+  public async getExportCommand(args: Array<string>) {
+    return await this.getCommand(['export', ...args]);
+  }
+
+  public async inspect(opts: InspectOpts): Promise<InspectResponse> {
+    const args: Array<string> = ['--format', 'json'];
+    if (opts.builder) {
+      args.push('--builder', opts.builder);
+    }
+    if (opts.ref) {
+      args.push(opts.ref);
+    }
+    const cmd = await this.getInspectCommand(args);
+    return await Exec.getExecOutput(cmd.command, cmd.args, {
+      ignoreReturnCode: true,
+      silent: true
+    }).then(res => {
+      if (res.stderr.length > 0 && res.exitCode != 0) {
+        throw new Error(res.stderr.trim());
+      }
+      return <InspectResponse>JSON.parse(res.stdout);
+    });
+  }
+
+  public async export(opts: ExportOpts): Promise<ExportResponse> {
     let builderName: string = '';
     let nodeName: string = '';
     const refs: Array<string> = [];
@@ -85,6 +101,72 @@ export class History {
     core.info(`exporting build record to ${outDir}`);
     fs.mkdirSync(outDir, {recursive: true});
 
+    if (opts.useContainer || (await this.buildx.versionSatisfies('<0.23.0'))) {
+      return await this.exportLegacy(builderName, nodeName, refs, outDir, opts.image);
+    }
+
+    // wait 3 seconds to ensure build records are finalized: https://github.com/moby/buildkit/pull/5109
+    await Util.sleep(3);
+
+    const summaries: Summaries = {};
+    if (!opts.noSummaries) {
+      for (const ref of refs) {
+        await this.inspect({
+          ref: ref,
+          builder: builderName
+        }).then(res => {
+          let errorLogs = '';
+          if (res.Error && res.Status !== 'canceled') {
+            if (res.Error.Message) {
+              errorLogs = res.Error.Message;
+            } else if (res.Error.Name && res.Error.Logs) {
+              errorLogs = `=> ${res.Error.Name}\n${res.Error.Logs}`;
+            }
+          }
+          summaries[ref] = {
+            name: res.Name,
+            status: res.Status,
+            duration: Util.formatDuration(res.Duration),
+            numCachedSteps: res.NumCachedSteps,
+            numTotalSteps: res.NumTotalSteps,
+            numCompletedSteps: res.NumCompletedSteps,
+            error: errorLogs
+          };
+        });
+      }
+    }
+
+    const dockerbuildPath = path.join(outDir, `${History.exportFilename(refs)}.dockerbuild`);
+
+    const cmd = await this.getExportCommand(['--builder', builderName, '--output', dockerbuildPath, ...refs]);
+    await Exec.getExecOutput(cmd.command, cmd.args);
+
+    const dockerbuildStats = fs.statSync(dockerbuildPath);
+
+    return {
+      dockerbuildFilename: dockerbuildPath,
+      dockerbuildSize: dockerbuildStats.size,
+      builderName: builderName,
+      nodeName: nodeName,
+      refs: refs,
+      summaries: summaries
+    };
+  }
+
+  private async exportLegacy(builderName: string, nodeName: string, refs: Array<string>, outDir: string, image?: string): Promise<ExportResponse> {
+    if (os.platform() === 'win32') {
+      throw new Error('Exporting a build record is currently not supported on Windows');
+    }
+    if (!(await Docker.isAvailable())) {
+      throw new Error('Docker is required to export a build record');
+    }
+    if (!(await Docker.isDaemonRunning())) {
+      throw new Error('Docker daemon needs to be running to export a build record');
+    }
+    if (!(await this.buildx.versionSatisfies('>=0.13.0'))) {
+      throw new Error('Buildx >= 0.13.0 is required to export a build record');
+    }
+
     // wait 3 seconds to ensure build records are finalized: https://github.com/moby/buildkit/pull/5109
     await Util.sleep(3);
 
@@ -139,7 +221,7 @@ export class History {
         'run', '--rm', '-i',
         '-v', `${Buildx.refsDir}:/buildx-refs`,
         '-v', `${outDir}:/out`,
-        opts.image || process.env[History.EXPORT_BUILD_IMAGE_ENV] || History.EXPORT_BUILD_IMAGE_DEFAULT,
+        image || process.env['DOCKER_BUILD_EXPORT_BUILD_IMAGE'] || 'docker.io/dockereng/export-build:latest',
         ...ebargs
       ]
       core.info(`[command]docker ${dockerRunArgs.join(' ')}`);
@@ -190,12 +272,7 @@ export class History {
         }
       });
 
-    let dockerbuildFilename = `${GitHub.context.repo.owner}~${GitHub.context.repo.repo}~${refs[0].substring(0, 6).toUpperCase()}`;
-    if (refs.length > 1) {
-      dockerbuildFilename += `+${refs.length - 1}`;
-    }
-
-    const dockerbuildPath = path.join(outDir, `${dockerbuildFilename}.dockerbuild`);
+    const dockerbuildPath = path.join(outDir, `${History.exportFilename(refs)}.dockerbuild`);
     fs.renameSync(tmpDockerbuildFilename, dockerbuildPath);
     const dockerbuildStats = fs.statSync(dockerbuildPath);
 
@@ -212,4 +289,12 @@ export class History {
       refs: refs
     };
   }
+
+  private static exportFilename(refs: Array<string>): string {
+    let name = `${GitHub.context.repo.owner}~${GitHub.context.repo.repo}~${refs[0].substring(0, 6).toUpperCase()}`;
+    if (refs.length > 1) {
+      name += `+${refs.length - 1}`;
+    }
+    return name;
+  }
 }

src/github.ts

@@ -269,56 +269,58 @@ export class GitHub {
     // Feedback survey
     sum.addRaw(`<p>`).addRaw(`Find this useful? `).addRaw(addLink('Let us know', 'https://docs.docker.com/feedback/gha-build-summary')).addRaw('</p>');
 
-    // Preview
-    sum.addRaw('<p>');
-    const summaryTableData: Array<Array<SummaryTableCell>> = [
-      [
-        {header: true, data: 'ID'},
-        {header: true, data: 'Name'},
-        {header: true, data: 'Status'},
-        {header: true, data: 'Cached'},
-        {header: true, data: 'Duration'}
-      ]
-    ];
-    let buildError: string | undefined;
-    for (const ref in opts.exportRes.summaries) {
-      if (Object.prototype.hasOwnProperty.call(opts.exportRes.summaries, ref)) {
-        const summary = opts.exportRes.summaries[ref];
-        // prettier-ignore
-        summaryTableData.push([
+    if (opts.exportRes.summaries) {
+      // Preview
+      sum.addRaw('<p>');
+      const summaryTableData: Array<Array<SummaryTableCell>> = [
+        [
+          {header: true, data: 'ID'},
+          {header: true, data: 'Name'},
+          {header: true, data: 'Status'},
+          {header: true, data: 'Cached'},
+          {header: true, data: 'Duration'}
+        ]
+      ];
+      let buildError: string | undefined;
+      for (const ref in opts.exportRes.summaries) {
+        if (Object.prototype.hasOwnProperty.call(opts.exportRes.summaries, ref)) {
+          const summary = opts.exportRes.summaries[ref];
+          // prettier-ignore
+          summaryTableData.push([
           {data: `<code>${ref.substring(0, 6).toUpperCase()}</code>`},
           {data: `<strong>${Util.stringToUnicodeEntities(summary.name)}</strong>`},
           {data: `${summary.status === 'completed' ? ':white_check_mark:' : summary.status === 'canceled' ? ':no_entry_sign:' : ':x:'} ${summary.status}`},
           {data: `${summary.numCachedSteps > 0 ? Math.round((summary.numCachedSteps / summary.numTotalSteps) * 100) : 0}%`},
           {data: summary.duration}
         ]);
-        if (summary.error) {
-          buildError = summary.error;
+          if (summary.error) {
+            buildError = summary.error;
+          }
         }
       }
-    }
-    sum.addTable([...summaryTableData]);
-    sum.addRaw(`</p>`);
+      sum.addTable([...summaryTableData]);
+      sum.addRaw(`</p>`);
 
-    // Build error
-    if (buildError) {
-      sum.addRaw(`<blockquote>`);
-      if (Util.countLines(buildError) > 10) {
-        // prettier-ignore
-        sum
+      // Build error
+      if (buildError) {
+        sum.addRaw(`<blockquote>`);
+        if (Util.countLines(buildError) > 10) {
+          // prettier-ignore
+          sum
           .addRaw(`<details><summary><strong>Error</strong></summary>`)
             .addCodeBlock(he.encode(buildError), 'text')
           .addRaw(`</details>`);
-      } else {
-        // prettier-ignore
-        sum
+        } else {
+          // prettier-ignore
+          sum
           .addRaw(`<strong>Error</strong>`)
           .addBreak()
           .addRaw(`<p>`)
             .addCodeBlock(he.encode(buildError), 'text')
           .addRaw(`</p>`);
+        }
+        sum.addRaw(`</blockquote>`);
       }
-      sum.addRaw(`</blockquote>`);
     }
 
     // Build inputs

src/types/buildx/history.ts

@@ -14,31 +14,122 @@
  * limitations under the License.
  */
 
-export interface ExportRecordOpts {
-  refs: Array<string>;
-  image?: string;
+export interface InspectOpts {
+  ref?: string;
+  builder?: string;
 }
 
-export interface ExportRecordResponse {
+export type BuildStatus = 'completed' | 'running' | 'failed' | 'canceled';
+
+export interface InspectResponse {
+  Name: string;
+  Ref: string;
+
+  Context?: string;
+  Dockerfile?: string;
+  VCSRepository?: string;
+  VCSRevision?: string;
+  Target?: string;
+  Platform?: Array<string>;
+  KeepGitDir?: boolean;
+
+  NamedContexts?: Array<InspectKeyValueOutput>;
+
+  StartedAt?: Date;
+  CompletedAt?: Date;
+  Duration: number;
+  Status: BuildStatus;
+  Error?: InspectErrorOutput;
+
+  NumCompletedSteps: number;
+  NumTotalSteps: number;
+  NumCachedSteps: number;
+
+  BuildArgs?: Array<InspectKeyValueOutput>;
+  Labels?: Array<InspectKeyValueOutput>;
+
+  Config?: InspectConfigOutput;
+
+  Materials?: InspectMaterialOutput[];
+  Attachments?: InspectAttachmentOutput[];
+
+  Errors?: Array<string>;
+}
+
+export interface InspectConfigOutput {
+  Network?: string;
+  ExtraHosts?: Array<string>;
+  Hostname?: string;
+  CgroupParent?: string;
+  ImageResolveMode?: string;
+  MultiPlatform?: boolean;
+  NoCache?: boolean;
+  NoCacheFilter?: Array<string>;
+
+  ShmSize?: string;
+  Ulimit?: string;
+  CacheMountNS?: string;
+  DockerfileCheckConfig?: string;
+  SourceDateEpoch?: string;
+  SandboxHostname?: string;
+
+  RestRaw?: Array<InspectKeyValueOutput>;
+}
+
+export interface InspectMaterialOutput {
+  URI?: string;
+  Digests?: Array<string>;
+}
+
+export interface InspectAttachmentOutput {
+  Digest?: string;
+  Platform?: string;
+  Type?: string;
+}
+
+export interface InspectErrorOutput {
+  Code?: number;
+  Message?: string;
+  Name?: string;
+  Logs?: Array<string>;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  Sources?: any;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  Stack?: any;
+}
+
+export interface InspectKeyValueOutput {
+  Name?: string;
+  Value?: string;
+}
+
+export interface ExportOpts {
+  refs: Array<string>;
+  noSummaries?: boolean;
+  image?: string;
+  useContainer?: boolean;
+}
+
+export interface ExportResponse {
   dockerbuildFilename: string;
   dockerbuildSize: number;
-  summaries: Summaries;
   builderName: string;
   nodeName: string;
   refs: Array<string>;
+  summaries?: Summaries;
 }
 
 export interface Summaries {
-  [ref: string]: RecordSummary;
+  [ref: string]: Summary;
 }
 
-export interface RecordSummary {
+export interface Summary {
   name: string;
   status: string;
   duration: string;
   numCachedSteps: number;
   numTotalSteps: number;
   numCompletedSteps: number;
-  frontendAttrs: Record<string, string>;
+  frontendAttrs?: Record<string, string>;
   error?: string;
 }

src/types/github.ts

@@ -19,7 +19,7 @@ import {components as OctoOpenApiTypes} from '@octokit/openapi-types';
 import {JwtPayload} from 'jwt-decode';
 
 import {BakeDefinition} from './buildx/bake';
-import {ExportRecordResponse} from './buildx/history';
+import {ExportResponse} from './buildx/history';
 
 export interface GitHubRelease {
   id: number;
@@ -57,7 +57,7 @@ export interface UploadArtifactResponse {
 }
 
 export interface BuildSummaryOpts {
-  exportRes: ExportRecordResponse;
+  exportRes: ExportResponse;
   uploadRes?: UploadArtifactResponse;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   inputs?: any;

src/util.ts

@@ -204,4 +204,20 @@ export class Util {
     const rcp = path.resolve(childPath);
     return rcp.startsWith(rpp.endsWith(path.sep) ? rpp : `${rpp}${path.sep}`);
   }
+
+  public static formatDuration(ns: number): string {
+    if (ns === 0) return '0s';
+
+    const totalSeconds = Math.floor(ns / 1e9);
+    const hours = Math.floor(totalSeconds / 3600);
+    const minutes = Math.floor((totalSeconds % 3600) / 60);
+    const seconds = totalSeconds % 60;
+
+    const parts: string[] = [];
+    if (hours) parts.push(`${hours}h`);
+    if (minutes) parts.push(`${minutes}m`);
+    if (seconds || parts.length === 0) parts.push(`${seconds}s`);
+
+    return parts.join('');
+  }
 }