Merge pull request #1016 from crazy-max/imagetools-inspect-retry

buildx(imagetools): make manifest retries configurable

__tests__/buildx/imagetools.test.itg.ts

@@ -30,12 +30,12 @@ const maybe = !process.env.GITHUB_ACTIONS || (process.env.GITHUB_ACTIONS === 'tr
 
 maybe('inspectImage', () => {
   it('inspect single platform', async () => {
-    const image = await new ImageTools().inspectImage('moby/buildkit:latest@sha256:5769c54b98840147b74128f38fb0b0a049e24b11a75bd81664131edd2854593f');
+    const image = await new ImageTools().inspectImage({name: 'moby/buildkit:latest@sha256:5769c54b98840147b74128f38fb0b0a049e24b11a75bd81664131edd2854593f'});
     const expectedImage = <Image>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-01.json'), {encoding: 'utf-8'}).trim());
     expect(image).toEqual(expectedImage);
   });
   it('inspect multi platform', async () => {
-    const image = await new ImageTools().inspectImage('moby/buildkit:latest@sha256:86c0ad9d1137c186e9d455912167df20e530bdf7f7c19de802e892bb8ca16552');
+    const image = await new ImageTools().inspectImage({name: 'moby/buildkit:latest@sha256:86c0ad9d1137c186e9d455912167df20e530bdf7f7c19de802e892bb8ca16552'});
     const expectedImage = <Record<string, Image>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-02.json'), {encoding: 'utf-8'}).trim());
     expect(image).toEqual(expectedImage);
   });
@@ -43,12 +43,12 @@ maybe('inspectImage', () => {
 
 maybe('inspectManifest', () => {
   it('inspect descriptor', async () => {
-    const manifest = await new ImageTools().inspectManifest('moby/buildkit:latest@sha256:dccc69dd895968c4f21aa9e43e715f25f0cedfce4b17f1014c88c307928e22fc');
+    const manifest = await new ImageTools().inspectManifest({name: 'moby/buildkit:latest@sha256:dccc69dd895968c4f21aa9e43e715f25f0cedfce4b17f1014c88c307928e22fc'});
     const expectedManifest = <Descriptor>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-03.json'), {encoding: 'utf-8'}).trim());
     expect(manifest).toEqual(expectedManifest);
   });
   it('inspect index', async () => {
-    const manifest = await new ImageTools().inspectManifest('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6');
+    const manifest = await new ImageTools().inspectManifest({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'});
     const expectedManifest = <ImageToolsManifest>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-04.json'), {encoding: 'utf-8'}).trim());
     expect(manifest).toEqual(expectedManifest);
   });
@@ -56,17 +56,17 @@ maybe('inspectManifest', () => {
 
 maybe('attestationDescriptors', () => {
   it('returns buildkit attestations descriptors', async () => {
-    const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6');
+    const attestations = await new ImageTools().attestationDescriptors({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'});
     const expectedAttestations = <Array<Descriptor>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-05.json'), {encoding: 'utf-8'}).trim());
     expect(attestations).toEqual(expectedAttestations);
   });
   it('returns buildkit attestations descriptors for linux/amd64', async () => {
-    const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'amd64'});
+    const attestations = await new ImageTools().attestationDescriptors({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'amd64'}});
     const expectedAttestations = <Array<Descriptor>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-06.json'), {encoding: 'utf-8'}).trim());
     expect(attestations).toEqual(expectedAttestations);
   });
   it('returns buildkit attestations descriptors for linux/arm/v7', async () => {
-    const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'arm', variant: 'v7'});
+    const attestations = await new ImageTools().attestationDescriptors({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'arm', variant: 'v7'}});
     const expectedAttestations = <Array<Descriptor>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-07.json'), {encoding: 'utf-8'}).trim());
     expect(attestations).toEqual(expectedAttestations);
   });
@@ -74,7 +74,7 @@ maybe('attestationDescriptors', () => {
 
 maybe('attestationDigests', () => {
   it('returns buildkit attestations digests', async () => {
-    const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6');
+    const digests = await new ImageTools().attestationDigests({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6'});
     // prettier-ignore
     expect(digests).toEqual([
       'sha256:2ba4ad6eae1efcafee73a971953093c7c32b6938f2f9fd4998c8bf4d0fbe76f2',
@@ -86,11 +86,11 @@ maybe('attestationDigests', () => {
     ]);
   });
   it('returns buildkit attestations digests for linux/amd64', async () => {
-    const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'amd64'});
+    const digests = await new ImageTools().attestationDigests({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'amd64'}});
     expect(digests).toEqual(['sha256:2ba4ad6eae1efcafee73a971953093c7c32b6938f2f9fd4998c8bf4d0fbe76f2']);
   });
   it('returns buildkit attestations digests for linux/arm/v7', async () => {
-    const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'arm', variant: 'v7'});
+    const digests = await new ImageTools().attestationDigests({name: 'moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', platform: {os: 'linux', architecture: 'arm', variant: 'v7'}});
     expect(digests).toEqual(['sha256:0709528fae1747ce17638ad2978ee7936b38a294136eaadaf692e415f64b1e03']);
   });
 });

__tests__/buildx/imagetools.test.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import {afterEach, describe, expect, it, vi} from 'vitest';
+import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
@@ -38,10 +38,133 @@ vi.spyOn(Context, 'tmpName').mockImplementation((): string => {
 });
 
 afterEach(() => {
+  vi.useRealTimers();
   vi.clearAllMocks();
   rimraf.sync(tmpDir);
 });
 
+beforeEach(() => {
+  vi.useRealTimers();
+  fs.mkdirSync(tmpDir, {recursive: true});
+});
+
+describe('inspectManifest', () => {
+  it('retries transient manifest unknown errors when requested', async () => {
+    vi.useFakeTimers();
+
+    const getCommand = vi.fn().mockResolvedValue({
+      command: 'docker',
+      args: ['buildx', 'imagetools', 'inspect']
+    });
+    const buildx = {getCommand} as unknown as Buildx;
+    const execSpy = vi
+      .spyOn(Exec, 'getExecOutput')
+      .mockResolvedValueOnce({
+        exitCode: 1,
+        stdout: '',
+        stderr: 'ERROR: MANIFEST_UNKNOWN: manifest unknown'
+      })
+      .mockResolvedValueOnce({
+        exitCode: 0,
+        stdout: JSON.stringify({
+          schemaVersion: 2,
+          mediaType: 'application/vnd.oci.image.index.v1+json',
+          manifests: []
+        }),
+        stderr: ''
+      });
+
+    const inspectPromise = new ImageTools({buildx}).inspectManifest({
+      name: 'docker.io/library/alpine:latest',
+      retryOnManifestUnknown: true,
+      retryLimit: 2
+    });
+
+    await vi.runAllTimersAsync();
+
+    expect(await inspectPromise).toEqual({
+      schemaVersion: 2,
+      mediaType: 'application/vnd.oci.image.index.v1+json',
+      manifests: []
+    });
+    expect(getCommand).toHaveBeenCalledWith(['imagetools', 'inspect', 'docker.io/library/alpine:latest', '--format', '{{json .Manifest}}']);
+    expect(execSpy).toHaveBeenCalledTimes(2);
+  });
+
+  it('does not retry non-manifest errors', async () => {
+    const getCommand = vi.fn().mockResolvedValue({
+      command: 'docker',
+      args: ['buildx', 'imagetools', 'inspect']
+    });
+    const buildx = {getCommand} as unknown as Buildx;
+    const execSpy = vi.spyOn(Exec, 'getExecOutput').mockResolvedValue({
+      exitCode: 1,
+      stdout: '',
+      stderr: 'ERROR: unauthorized'
+    });
+
+    const result = await new ImageTools({buildx})
+      .inspectManifest({
+        name: 'docker.io/library/alpine:latest',
+        retryOnManifestUnknown: true
+      })
+      .then(
+        value => ({value, error: undefined}),
+        error => ({value: undefined, error: error as Error})
+      );
+
+    expect(result.value).toBeUndefined();
+    expect(result.error).toBeInstanceOf(Error);
+    expect(result.error?.message).toContain('ERROR: unauthorized');
+
+    expect(execSpy).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe('inspectImage', () => {
+  it('retries transient manifest unknown errors when requested', async () => {
+    vi.useFakeTimers();
+
+    const getCommand = vi.fn().mockResolvedValue({
+      command: 'docker',
+      args: ['buildx', 'imagetools', 'inspect']
+    });
+    const buildx = {getCommand} as unknown as Buildx;
+    const execSpy = vi
+      .spyOn(Exec, 'getExecOutput')
+      .mockResolvedValueOnce({
+        exitCode: 1,
+        stdout: '',
+        stderr: 'ERROR: MANIFEST_UNKNOWN: manifest unknown'
+      })
+      .mockResolvedValueOnce({
+        exitCode: 0,
+        stdout: JSON.stringify({
+          config: {
+            digest: 'sha256:test'
+          }
+        }),
+        stderr: ''
+      });
+
+    const inspectPromise = new ImageTools({buildx}).inspectImage({
+      name: 'docker.io/library/alpine:latest',
+      retryOnManifestUnknown: true,
+      retryLimit: 2
+    });
+
+    await vi.runAllTimersAsync();
+
+    expect(await inspectPromise).toEqual({
+      config: {
+        digest: 'sha256:test'
+      }
+    });
+    expect(getCommand).toHaveBeenCalledWith(['imagetools', 'inspect', 'docker.io/library/alpine:latest', '--format', '{{json .Image}}']);
+    expect(execSpy).toHaveBeenCalledTimes(2);
+  });
+});
+
 describe('create', () => {
   it('parses metadata and supports cwd sources', async () => {
     const getCommand = vi.fn().mockResolvedValue({

src/buildx/imagetools.ts

@@ -21,9 +21,9 @@ import {Buildx} from './buildx.js';
 import {Context} from '../context.js';
 import {Exec} from '../exec.js';
 
-import {CreateOpts, CreateResponse, CreateResult, Manifest as ImageToolsManifest} from '../types/buildx/imagetools.js';
+import {AttestationInspectOpts, CreateOpts, CreateResponse, CreateResult, InspectOpts, Manifest as ImageToolsManifest} from '../types/buildx/imagetools.js';
 import {Image} from '../types/oci/config.js';
-import {Descriptor, Platform} from '../types/oci/descriptor.js';
+import {Descriptor} from '../types/oci/descriptor.js';
 import {Digest} from '../types/oci/digest.js';
 
 export interface ImageToolsOpts {
@@ -49,16 +49,8 @@ export class ImageTools {
     return await this.getCommand(['create', ...args]);
   }
 
-  public async inspectImage(name: string): Promise<Record<string, Image> | Image> {
+  public async inspectImage(opts: InspectOpts): Promise<Record<string, Image> | Image> {
-    const cmd = await this.getInspectCommand([name, '--format', '{{json .Image}}']);
+    return await this.inspect(opts, '{{json .Image}}', parsedOutput => {
-    return await Exec.getExecOutput(cmd.command, cmd.args, {
-      ignoreReturnCode: true,
-      silent: true
-    }).then(res => {
-      if (res.stderr.length > 0 && res.exitCode != 0) {
-        throw new Error(res.stderr.trim());
-      }
-      const parsedOutput = JSON.parse(res.stdout);
       if (typeof parsedOutput === 'object' && !Array.isArray(parsedOutput) && parsedOutput !== null) {
         if (Object.prototype.hasOwnProperty.call(parsedOutput, 'config')) {
           return <Image>parsedOutput;
@@ -70,16 +62,8 @@ export class ImageTools {
     });
   }
 
-  public async inspectManifest(name: string): Promise<ImageToolsManifest | Descriptor> {
+  public async inspectManifest(opts: InspectOpts): Promise<ImageToolsManifest | Descriptor> {
-    const cmd = await this.getInspectCommand([name, '--format', '{{json .Manifest}}']);
+    return await this.inspect(opts, '{{json .Manifest}}', parsedOutput => {
-    return await Exec.getExecOutput(cmd.command, cmd.args, {
-      ignoreReturnCode: true,
-      silent: true
-    }).then(res => {
-      if (res.stderr.length > 0 && res.exitCode != 0) {
-        throw new Error(res.stderr.trim());
-      }
-      const parsedOutput = JSON.parse(res.stdout);
       if (typeof parsedOutput === 'object' && !Array.isArray(parsedOutput) && parsedOutput !== null) {
         if (Object.prototype.hasOwnProperty.call(parsedOutput, 'manifests')) {
           return <ImageToolsManifest>parsedOutput;
@@ -91,17 +75,18 @@ export class ImageTools {
     });
   }
 
-  public async attestationDescriptors(name: string, platform?: Platform): Promise<Array<Descriptor>> {
+  public async attestationDescriptors(opts: AttestationInspectOpts): Promise<Array<Descriptor>> {
-    const manifest = await this.inspectManifest(name);
+    const manifest = await this.inspectManifest(opts);
 
     if (typeof manifest !== 'object' || manifest === null || !('manifests' in manifest) || !Array.isArray(manifest.manifests)) {
-      throw new Error(`No descriptor found for ${name}`);
+      throw new Error(`No descriptor found for ${opts.name}`);
     }
 
     const attestations = manifest.manifests.filter(m => m.annotations?.['vnd.docker.reference.type'] === 'attestation-manifest');
-    if (!platform) {
+    if (!opts.platform) {
       return attestations;
     }
+    const platform = opts.platform;
 
     const manifestByDigest = new Map<string, Descriptor>();
     for (const m of manifest.manifests) {
@@ -123,8 +108,8 @@ export class ImageTools {
     });
   }
 
-  public async attestationDigests(name: string, platform?: Platform): Promise<Array<Digest>> {
+  public async attestationDigests(opts: AttestationInspectOpts): Promise<Array<Digest>> {
-    return (await this.attestationDescriptors(name, platform)).map(attestation => attestation.digest);
+    return (await this.attestationDescriptors(opts)).map(attestation => attestation.digest);
   }
 
   public async create(opts: CreateOpts): Promise<CreateResult | undefined> {
@@ -205,4 +190,44 @@ export class ImageTools {
       }
     });
   }
+
+  private async inspect<T>(opts: InspectOpts, format: string, parser: (parsedOutput: unknown) => T): Promise<T> {
+    const cmd = await this.getInspectCommand([opts.name, '--format', format]);
+    if (!opts.retryOnManifestUnknown) {
+      return await this.execInspect(cmd.command, cmd.args, parser);
+    }
+
+    const retries = opts.retryLimit ?? 15;
+    let lastError: Error | undefined;
+    for (let attempt = 0; attempt < retries; attempt++) {
+      try {
+        return await this.execInspect(cmd.command, cmd.args, parser);
+      } catch (err) {
+        lastError = err as Error;
+        if (!ImageTools.isManifestUnknownError(lastError.message) || attempt === retries - 1) {
+          throw lastError;
+        }
+        core.info(`buildx imagetools inspect command failed with MANIFEST_UNKNOWN, retrying attempt ${attempt + 1}/${retries}...\n${lastError.message}`);
+        await new Promise(res => setTimeout(res, Math.pow(2, attempt) * 100));
+      }
+    }
+
+    throw lastError ?? new Error(`ImageTools inspect command failed for ${opts.name}`);
+  }
+
+  private async execInspect<T>(command: string, args: Array<string>, parser: (parsedOutput: unknown) => T): Promise<T> {
+    return await Exec.getExecOutput(command, args, {
+      ignoreReturnCode: true,
+      silent: true
+    }).then(res => {
+      if (res.stderr.length > 0 && res.exitCode != 0) {
+        throw new Error(res.stderr.trim());
+      }
+      return parser(JSON.parse(res.stdout));
+    });
+  }
+
+  private static isManifestUnknownError(message: string): boolean {
+    return /(MANIFEST_UNKNOWN|manifest unknown|not found: not found)/i.test(message);
+  }
 }

src/sigstore/sigstore.ts

@@ -113,7 +113,11 @@ export class Sigstore {
       }
 
       for (const imageName of opts.imageNames) {
-        const attestationDigests = await this.imageTools.attestationDigests(`${imageName}@${opts.imageDigest}`);
+        const attestationDigests = await this.imageTools.attestationDigests({
+          name: `${imageName}@${opts.imageDigest}`,
+          retryOnManifestUnknown: opts.retryOnManifestUnknown,
+          retryLimit: opts.retryLimit
+        });
         for (const attestationDigest of attestationDigests) {
           const attestationRef = `${imageName}@${attestationDigest}`;
           await core.group(`Signing attestation manifest ${attestationRef}`, async () => {
@@ -183,7 +187,12 @@ export class Sigstore {
   public async verifyImageAttestations(image: string, opts: VerifySignedManifestsOpts): Promise<Record<string, VerifySignedManifestsResult>> {
     const result: Record<string, VerifySignedManifestsResult> = {};
 
-    const attestationDigests = await this.imageTools.attestationDigests(image, opts.platform);
+    const attestationDigests = await this.imageTools.attestationDigests({
+      name: image,
+      platform: opts.platform,
+      retryOnManifestUnknown: opts.retryOnManifestUnknown,
+      retryLimit: opts.retryLimit
+    });
     if (attestationDigests.length === 0) {
       throw new Error(`No attestation manifests found for ${image}`);
     }
@@ -237,7 +246,7 @@ export class Sigstore {
       };
     }
 
-    const retries = 15;
+    const retries = opts.retryLimit ?? 15;
     let lastError: Error | undefined;
     core.info(`[command]cosign ${[...cosignArgs, attestationRef].join(' ')}`);
     for (let attempt = 0; attempt < retries; attempt++) {

src/types/buildx/imagetools.ts

@@ -15,9 +15,19 @@
  */
 
 import {Versioned} from '../oci/versioned.js';
-import {Descriptor} from '../oci/descriptor.js';
+import {Descriptor, Platform} from '../oci/descriptor.js';
 import {Digest} from '../oci/digest.js';
 
+export interface InspectOpts {
+  name: string;
+  retryOnManifestUnknown?: boolean;
+  retryLimit?: number;
+}
+
+export interface AttestationInspectOpts extends InspectOpts {
+  platform?: Platform;
+}
+
 // https://github.com/docker/buildx/blob/62857022a08552bee5cad0c3044a9a3b185f0b32/util/imagetools/printers.go#L109-L123
 export interface Manifest extends Versioned {
   mediaType?: string;

src/types/sigstore/sigstore.ts

@@ -40,6 +40,8 @@ export interface SignAttestationManifestsOpts {
   imageNames: Array<string>;
   imageDigest: string;
   noTransparencyLog?: boolean;
+  retryOnManifestUnknown?: boolean;
+  retryLimit?: number;
 }
 
 export interface SignAttestationManifestsResult extends ParsedBundle {
@@ -51,6 +53,7 @@ export interface VerifySignedManifestsOpts {
   platform?: Platform;
   noTransparencyLog?: boolean;
   retryOnManifestUnknown?: boolean;
+  retryLimit?: number;
 }
 
 export interface VerifySignedManifestsResult {