Merge pull request #938 from crazy-max/bake-def-envs

buildx(bake): merge existing env vars when parsing definition

__tests__/.fixtures/imagetools-06.json

@@ -0,0 +1,15 @@
+[
+  {
+    "mediaType":"application/vnd.oci.image.manifest.v1+json",
+    "digest":"sha256:2ba4ad6eae1efcafee73a971953093c7c32b6938f2f9fd4998c8bf4d0fbe76f2",
+    "size":1113,
+    "annotations":{
+      "vnd.docker.reference.digest":"sha256:dccc69dd895968c4f21aa9e43e715f25f0cedfce4b17f1014c88c307928e22fc",
+      "vnd.docker.reference.type":"attestation-manifest"
+    },
+    "platform":{
+      "architecture":"unknown",
+      "os":"unknown"
+    }
+  }
+]

__tests__/.fixtures/imagetools-07.json

@@ -0,0 +1,15 @@
+[
+  {
+    "mediaType": "application/vnd.oci.image.manifest.v1+json",
+    "digest": "sha256:0709528fae1747ce17638ad2978ee7936b38a294136eaadaf692e415f64b1e03",
+    "size": 1113,
+    "annotations": {
+      "vnd.docker.reference.digest": "sha256:1b6bce668653f08e2d0f9f7c9b646675b2cbce94ce8abdf4eb0eabaef4353045",
+      "vnd.docker.reference.type": "attestation-manifest"
+    },
+    "platform": {
+      "architecture": "unknown",
+      "os": "unknown"
+    }
+  }
+]

__tests__/buildx/imagetools.test.itg.ts

@@ -60,6 +60,16 @@ maybe('attestationDescriptors', () => {
     const expectedAttestations = <Array<Descriptor>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-05.json'), {encoding: 'utf-8'}).trim());
     expect(attestations).toEqual(expectedAttestations);
   });
+  it('returns buildkit attestations descriptors for linux/amd64', async () => {
+    const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'amd64'});
+    const expectedAttestations = <Array<Descriptor>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-06.json'), {encoding: 'utf-8'}).trim());
+    expect(attestations).toEqual(expectedAttestations);
+  });
+  it('returns buildkit attestations descriptors for linux/arm/v7', async () => {
+    const attestations = await new ImageTools().attestationDescriptors('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'arm', variant: 'v7'});
+    const expectedAttestations = <Array<Descriptor>>JSON.parse(fs.readFileSync(path.join(fixturesDir, 'imagetools-07.json'), {encoding: 'utf-8'}).trim());
+    expect(attestations).toEqual(expectedAttestations);
+  });
 });
 
 maybe('attestationDigests', () => {
@@ -75,4 +85,12 @@ maybe('attestationDigests', () => {
       'sha256:d95ca72d4f2a6bc416d4b2f3003b2af9d5f4dea99acec6ad3ab0c2082000a98c'
     ]);
   });
+  it('returns buildkit attestations digests for linux/amd64', async () => {
+    const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'amd64'});
+    expect(digests).toEqual(['sha256:2ba4ad6eae1efcafee73a971953093c7c32b6938f2f9fd4998c8bf4d0fbe76f2']);
+  });
+  it('returns buildkit attestations digests for linux/arm/v7', async () => {
+    const digests = await new ImageTools().attestationDigests('moby/buildkit:latest@sha256:79cc6476ab1a3371c9afd8b44e7c55610057c43e18d9b39b68e2b0c2475cc1b6', {os: 'linux', architecture: 'arm', variant: 'v7'});
+    expect(digests).toEqual(['sha256:0709528fae1747ce17638ad2978ee7936b38a294136eaadaf692e415f64b1e03']);
+  });
 });

__tests__/oci/oci.test.ts

@@ -14,14 +14,17 @@
  * limitations under the License.
  */
 
-import {afterEach, describe, expect, test} from '@jest/globals';
+import {afterEach, describe, expect, jest, test} from '@jest/globals';
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
 import * as rimraf from 'rimraf';
+import osm = require('os');
 
 import {OCI} from '../../src/oci/oci';
 
+import {Platform} from '../../src/types/oci/descriptor';
+
 const fixturesDir = path.join(__dirname, '..', '.fixtures');
 const tmpDir = fs.mkdtempSync(path.join(process.env.TEMP || os.tmpdir(), 'oci-oci-'));
 
@@ -29,6 +32,25 @@ afterEach(function () {
   rimraf.sync(tmpDir);
 });
 
+describe('defaultPlatform', () => {
+  test.each([
+    ['win32', 'x64', {architecture: 'amd64', os: 'windows'}],
+    ['win32', 'arm64', {architecture: 'arm64', os: 'windows'}],
+    ['darwin', 'x64', {architecture: 'amd64', os: 'darwin'}],
+    ['darwin', 'arm64', {architecture: 'arm64', os: 'darwin'}],
+    ['linux', 'ia32', {architecture: '386', os: 'linux'}],
+    ['linux', 'x64', {architecture: 'amd64', os: 'linux'}],
+    ['linux', 'arm64', {architecture: 'arm64', os: 'linux'}],
+    ['linux', 'ppc64', {architecture: 'ppc64le', os: 'linux'}],
+    ['linux', 's390x', {architecture: 's390x', os: 'linux'}]
+  ])('default platform for %s/%s', async (os: string, arch: string, expected: Platform) => {
+    jest.spyOn(osm, 'platform').mockImplementation(() => os as NodeJS.Platform);
+    jest.spyOn(osm, 'arch').mockImplementation(() => arch);
+    const res = OCI.defaultPlatform();
+    expect(res).toEqual(expected);
+  });
+});
+
 describe('loadArchive', () => {
   // prettier-ignore
   test.each(fs.readdirSync(path.join(fixturesDir, 'oci-archive')).filter(file => {

__tests__/sigstore/sigstore.test.itg.ts

@@ -23,6 +23,7 @@ import {Build} from '../../src/buildx/build';
 import {Install as CosignInstall} from '../../src/cosign/install';
 import {Docker} from '../../src/docker/docker';
 import {Exec} from '../../src/exec';
+import {OCI} from '../../src/oci/oci';
 import {Sigstore} from '../../src/sigstore/sigstore';
 
 const fixturesDir = path.join(__dirname, '..', '.fixtures');
@@ -114,6 +115,20 @@ maybe('verifyImageAttestations', () => {
     },
     60000
   );
+
+  it('default platform', async () => {
+    const sigstore = new Sigstore();
+    const verifyResults = await sigstore.verifyImageAttestations('moby/buildkit:master@sha256:84014da3581b2ff2c14cb4f60029cf9caa272b79e58f2e89c651ea6966d7a505', {
+      certificateIdentityRegexp: `^https://github.com/docker/github-builder-experimental/.github/workflows/bake.yml.*$`,
+      platform: OCI.defaultPlatform()
+    });
+    expect(Object.keys(verifyResults).length).toEqual(1);
+    for (const [attestationRef, res] of Object.entries(verifyResults)) {
+      expect(attestationRef).toBeDefined();
+      expect(res.cosignArgs).toBeDefined();
+      expect(res.signatureManifestDigest).toBeDefined();
+    }
+  });
 });
 
 maybeIdToken('signProvenanceBlobs', () => {

src/buildx/bake.ts

@@ -105,13 +105,7 @@ export class Bake {
   public async getDefinition(cmdOpts: BakeCmdOpts, execOptions?: ExecOptions): Promise<BakeDefinition> {
     execOptions = execOptions || {ignoreReturnCode: true};
     execOptions.ignoreReturnCode = true;
-    if (cmdOpts.githubToken) {
-      execOptions.env = Object.assign({}, process.env, {
-        BUILDX_BAKE_GIT_AUTH_TOKEN: cmdOpts.githubToken
-      }) as {
-        [key: string]: string;
-      };
-    }
+    execOptions.env = Object.assign({}, process.env, execOptions.env || {}, cmdOpts.githubToken ? {BUILDX_BAKE_GIT_AUTH_TOKEN: cmdOpts.githubToken} : {});
 
     const args = ['bake'];
 

src/buildx/imagetools.ts

@@ -19,7 +19,7 @@ import {Exec} from '../exec';
 
 import {Manifest as ImageToolsManifest} from '../types/buildx/imagetools';
 import {Image} from '../types/oci/config';
-import {Descriptor} from '../types/oci/descriptor';
+import {Descriptor, Platform} from '../types/oci/descriptor';
 import {Digest} from '../types/oci/digest';
 
 export interface ImageToolsOpts {
@@ -83,15 +83,39 @@ export class ImageTools {
     });
   }
 
-  public async attestationDescriptors(name: string): Promise<Array<Descriptor>> {
+  public async attestationDescriptors(name: string, platform?: Platform): Promise<Array<Descriptor>> {
     const manifest = await this.inspectManifest(name);
-    if (typeof manifest === 'object' && manifest !== null && 'manifests' in manifest && Array.isArray(manifest.manifests)) {
-      return manifest.manifests.filter(m => m.annotations && m.annotations['vnd.docker.reference.type'] === 'attestation-manifest');
+
+    if (typeof manifest !== 'object' || manifest === null || !('manifests' in manifest) || !Array.isArray(manifest.manifests)) {
+      throw new Error(`No descriptor found for ${name}`);
     }
-    throw new Error(`No attestation descriptors found for ${name}`);
+
+    const attestations = manifest.manifests.filter(m => m.annotations?.['vnd.docker.reference.type'] === 'attestation-manifest');
+    if (!platform) {
+      return attestations;
+    }
+
+    const manifestByDigest = new Map<string, Descriptor>();
+    for (const m of manifest.manifests) {
+      if (m.digest) {
+        manifestByDigest.set(m.digest, m);
+      }
+    }
+
+    return attestations.filter(attestation => {
+      const refDigest = attestation.annotations?.['vnd.docker.reference.digest'];
+      if (!refDigest) {
+        return false;
+      }
+      const referencedManifest = manifestByDigest.get(refDigest);
+      if (!referencedManifest) {
+        return false;
+      }
+      return referencedManifest.platform?.os === platform.os && referencedManifest.platform?.architecture === platform.architecture && (referencedManifest.platform?.variant ?? '') === (platform.variant ?? '');
+    });
   }
 
-  public async attestationDigests(name: string): Promise<Array<Digest>> {
-    return (await this.attestationDescriptors(name)).map(attestation => attestation.digest);
+  public async attestationDigests(name: string, platform?: Platform): Promise<Array<Digest>> {
+    return (await this.attestationDescriptors(name, platform)).map(attestation => attestation.digest);
   }
 }

src/oci/oci.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 import fs from 'fs';
+import os from 'os';
 import gunzip from 'gunzip-maybe';
 import * as path from 'path';
 import {Readable} from 'stream';
@@ -21,12 +22,59 @@ import * as tar from 'tar-stream';
 
 import {Archive, LoadArchiveOpts} from '../types/oci/oci';
 import {Index} from '../types/oci';
+import {Platform} from '../types/oci/descriptor';
 import {Manifest} from '../types/oci/manifest';
 import {Image} from '../types/oci/config';
 import {IMAGE_BLOBS_DIR_V1, IMAGE_INDEX_FILE_V1, IMAGE_LAYOUT_FILE_V1, ImageLayout} from '../types/oci/layout';
 import {MEDIATYPE_IMAGE_INDEX_V1, MEDIATYPE_IMAGE_MANIFEST_V1} from '../types/oci/mediatype';
 
 export class OCI {
+  public static defaultPlatform(): Platform {
+    const nodePlatform = os.platform();
+    const nodeArch = os.arch();
+
+    const goosMap: Record<string, string> = {
+      win32: 'windows',
+      sunos: 'solaris'
+      // others (linux, darwin, freebsd, openbsd, netbsd, aix, android) match Go already
+    };
+
+    const goArchMap: Record<string, string> = {
+      x64: 'amd64',
+      ia32: '386',
+      arm: 'arm',
+      arm64: 'arm64',
+      ppc64: 'ppc64le',
+      s390x: 's390x',
+      riscv64: 'riscv64',
+      loong64: 'loong64',
+      mips: 'mips',
+      mipsel: 'mipsle',
+      mips64: 'mips64',
+      mips64el: 'mips64le'
+    };
+
+    const goos = goosMap[nodePlatform] ?? nodePlatform;
+    const goarch = goArchMap[nodeArch] ?? nodeArch;
+
+    let variant: string | undefined;
+    if (goarch === 'arm') {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const armVersionRaw = (process.config.variables as any)?.arm_version;
+      const armVersion = Number(armVersionRaw);
+      // Go only recognizes v5/v6/v7 for GOARM. Do not emit v8+ (that would be arm64).
+      if ([5, 6, 7].includes(armVersion)) {
+        variant = `v${armVersion}`;
+      }
+    }
+
+    return {
+      architecture: goarch,
+      os: goos,
+      variant: variant
+    };
+  }
+
   public static loadArchive(opts: LoadArchiveOpts): Promise<Archive> {
     return new Promise<Archive>((resolve, reject) => {
       const tarex: tar.Extract = tar.extract();

src/sigstore/sigstore.ts

@@ -133,9 +133,9 @@ export class Sigstore {
     for (const [attestationRef, signedRes] of Object.entries(signedManifestsResult)) {
       await core.group(`Verifying signature of ${attestationRef}`, async () => {
         const verifyResult = await this.verifyImageAttestation(attestationRef, {
-          noTransparencyLog: opts.noTransparencyLog || !signedRes.tlogID,
           certificateIdentityRegexp: opts.certificateIdentityRegexp,
-          retries: opts.retries
+          noTransparencyLog: opts.noTransparencyLog || !signedRes.tlogID,
+          retryOnManifestUnknown: opts.retryOnManifestUnknown
         });
         core.info(`Signature manifest verified: https://oci.dag.dev/?image=${signedRes.imageName}@${verifyResult.signatureManifestDigest}`);
         result[attestationRef] = verifyResult;
@@ -147,7 +147,7 @@ export class Sigstore {
   public async verifyImageAttestations(image: string, opts: VerifySignedManifestsOpts): Promise<Record<string, VerifySignedManifestsResult>> {
     const result: Record<string, VerifySignedManifestsResult> = {};
 
-    const attestationDigests = await this.imageTools.attestationDigests(image);
+    const attestationDigests = await this.imageTools.attestationDigests(image, opts.platform);
     if (attestationDigests.length === 0) {
       throw new Error(`No attestation manifests found for ${image}`);
     }
@@ -164,8 +164,6 @@ export class Sigstore {
   }
 
   public async verifyImageAttestation(attestationRef: string, opts: VerifySignedManifestsOpts): Promise<VerifySignedManifestsResult> {
-    const retries = opts.retries ?? 15;
-
     if (!(await this.cosign.isAvailable())) {
       throw new Error('Cosign is required to verify signed manifests');
     }
@@ -183,6 +181,27 @@ export class Sigstore {
       cosignArgs.push('--use-signed-timestamps', '--insecure-ignore-tlog');
     }
 
+    if (!opts.retryOnManifestUnknown) {
+      core.info(`[command]cosign ${[...cosignArgs, attestationRef].join(' ')}`);
+      const execRes = await Exec.getExecOutput('cosign', ['--verbose', ...cosignArgs, attestationRef], {
+        ignoreReturnCode: true,
+        silent: true,
+        env: Object.assign({}, process.env, {
+          COSIGN_EXPERIMENTAL: '1'
+        }) as {[key: string]: string}
+      });
+      if (execRes.exitCode !== 0) {
+        // prettier-ignore
+        throw new Error(`Cosign verify command failed with: ${execRes.stderr.trim().split(/\r?\n/).filter(line => line.length > 0).pop() ?? 'unknown error'}`);
+      }
+      const verifyResult = Cosign.parseCommandOutput(execRes.stderr.trim());
+      return {
+        cosignArgs: cosignArgs,
+        signatureManifestDigest: verifyResult.signatureManifestDigest!
+      };
+    }
+
+    const retries = 15;
     let lastError: Error | undefined;
     core.info(`[command]cosign ${[...cosignArgs, attestationRef].join(' ')}`);
     for (let attempt = 0; attempt < retries; attempt++) {

src/types/sigstore/sigstore.ts

@@ -17,6 +17,7 @@
 import type {SerializedBundle} from '@sigstore/bundle';
 
 import {Subject} from '../intoto/intoto';
+import {Platform} from '../oci/descriptor';
 
 export const FULCIO_URL = 'https://fulcio.sigstore.dev';
 export const REKOR_URL = 'https://rekor.sigstore.dev';
@@ -47,8 +48,9 @@ export interface SignAttestationManifestsResult extends ParsedBundle {
 
 export interface VerifySignedManifestsOpts {
   certificateIdentityRegexp: string;
+  platform?: Platform;
   noTransparencyLog?: boolean;
-  retries?: number;
+  retryOnManifestUnknown?: boolean;
 }
 
 export interface VerifySignedManifestsResult {