Merge pull request #496 from crazy-max/ci-split-docker-install-tests

ci: split docker install integration tests

.github/buildx-releases.json

@@ -40,6 +40,59 @@
       "https://github.com/docker/buildx/releases/download/v0.18.0/checksums.txt"
     ]
   },
+  "v0.19.0-rc1": {
+    "id": 186764141,
+    "tag_name": "v0.19.0-rc1",
+    "html_url": "https://github.com/docker/buildx/releases/tag/v0.19.0-rc1",
+    "assets": [
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.darwin-amd64",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.darwin-amd64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.darwin-amd64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.darwin-arm64",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.darwin-arm64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.darwin-arm64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.freebsd-amd64",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.freebsd-amd64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.freebsd-amd64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.freebsd-arm64",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.freebsd-arm64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.freebsd-arm64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-amd64",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-amd64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-amd64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-arm-v6",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-arm-v6.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-arm-v6.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-arm-v7",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-arm-v7.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-arm-v7.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-arm64",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-arm64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-arm64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-ppc64le",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-ppc64le.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-ppc64le.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-riscv64",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-riscv64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-riscv64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-s390x",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-s390x.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.linux-s390x.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.openbsd-amd64",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.openbsd-amd64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.openbsd-amd64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.openbsd-arm64",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.openbsd-arm64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.openbsd-arm64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.windows-amd64.exe",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.windows-amd64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.windows-amd64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.windows-arm64.exe",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.windows-arm64.provenance.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/buildx-v0.19.0-rc1.windows-arm64.sbom.json",
+      "https://github.com/docker/buildx/releases/download/v0.19.0-rc1/checksums.txt"
+    ]
+  },
   "v0.18.0": {
     "id": 182806222,
     "tag_name": "v0.18.0",

.github/docker-releases.json

@@ -5,6 +5,12 @@
     "html_url": "https://github.com/moby/moby/releases/tag/v27.3.1",
     "assets": []
   },
+  "v27.4.0-rc.2": {
+    "id": 186229632,
+    "tag_name": "v27.4.0-rc.2",
+    "html_url": "https://github.com/moby/moby/releases/tag/v27.4.0-rc.2",
+    "assets": []
+  },
   "v27.4.0-rc.1": {
     "id": 185528936,
     "tag_name": "v27.4.0-rc.1",

.github/workflows/build.yml

@@ -15,6 +15,12 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        node_version:
+          - 20
+          - 18
     steps:
       -
         name: Checkout
@@ -24,3 +30,5 @@ jobs:
         uses: docker/bake-action@v5
         with:
           targets: build
+        env:
+          NODE_VERSION: ${{ matrix.node_version }}

.github/workflows/test.yml

@@ -16,11 +16,17 @@ on:
 env:
   NODE_VERSION: "20"
   BUILDX_VERSION: "v0.18.0"
-  BUILDKIT_IMAGE: "moby/buildkit:v0.17.0"
+  BUILDKIT_IMAGE: "moby/buildkit:v0.17.2"
 
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        node_version:
+          - 20
+          - 18
     steps:
       -
         name: Checkout
@@ -32,6 +38,7 @@ jobs:
           targets: test-coverage
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NODE_VERSION: ${{ matrix.node_version }}
       -
         name: Check coverage
         run: |
@@ -44,7 +51,7 @@ jobs:
       -
         name: Upload coverage
         uses: codecov/codecov-action@v5
-        if: env.RUN_CODECOV == 'true'
+        if: env.RUN_CODECOV == 'true' && matrix.node_version == env.NODE_VERSION
         with:
           files: ./coverage/clover.xml
           flags: unit
@@ -53,7 +60,7 @@ jobs:
   prepare-itg:
     runs-on: ubuntu-latest
     outputs:
-      matrix: ${{ steps.tests.outputs.matrix }}
+      includes: ${{ steps.set.outputs.includes }}
     steps:
       -
         name: Checkout
@@ -73,14 +80,43 @@ jobs:
         name: Install
         run: yarn install
       -
-        name: Create matrix
-        id: tests
-        run: |
-          declare -a tests
-          for test in $(yarn run test:itg-list); do
-            tests+=("${test#$(pwd)/__tests__/}")
-          done
-          echo "matrix=$(echo ${tests[@]} | jq -cR 'split(" ")')" >>${GITHUB_OUTPUT}
+        name: Create includes
+        id: set
+        uses: actions/github-script@v7
+        with:
+          script: |
+            let tests = [];
+            await core.group(`Get tests`, async () => {
+              const res = await exec.getExecOutput('yarn', ['run', 'test:itg-list'], {
+                silent: true,
+                ignoreReturnCode: true
+              });
+              if (res.stderr.length > 0 && res.exitCode != 0) {
+                throw new Error(res.stderr);
+              }
+              for (const test of res.stdout.trim().split('\n')) {
+                tests.push(test.replace(/^.*__tests__\//, ''));
+              }
+              core.info(`tests: ${JSON.stringify(tests)}`);
+            });
+            await core.group(`Set includes`, async () => {
+              let includes = [];
+              for (const os of ['ubuntu-latest', 'macos-13', 'windows-latest']) {
+                for (const test of tests) {
+                  if (os === 'macos-13' && test === 'docker/install.test.itg.ts') {
+                    includes.push({ os: os, test: test, docker_install_type: 'image', docker_install_version: '27.3.1' });
+                    includes.push({ os: os, test: test, docker_install_type: 'image', docker_install_version: 'master' });
+                    includes.push({ os: os, test: test, docker_install_type: 'image', docker_install_version: 'latest' });
+                    includes.push({ os: os, test: test, docker_install_type: 'archive', docker_install_version: 'v26.1.4' });
+                    includes.push({ os: os, test: test, docker_install_type: 'archive', docker_install_version: 'latest' });
+                  } else {
+                    includes.push({ os: os, test: test });
+                  }
+                }
+              }
+              core.info(`includes: ${JSON.stringify(includes)}`);
+              core.setOutput('includes', JSON.stringify(includes));
+            });
       -
         name: Show matrix
         run: |
@@ -93,12 +129,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: ${{ fromJson(needs.prepare-itg.outputs.matrix) }}
-        os:
-          - ubuntu-latest
-          #- macos-14  # no virt: https://github.com/docker/actions-toolkit/issues/317
-          - macos-13
-          - windows-latest
+        include: ${{ fromJson(needs.prepare-itg.outputs.includes) }}
     steps:
       -
         name: Checkout
@@ -151,6 +182,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CTN_BUILDER_NAME: ${{ steps.builder.outputs.name }}
           TEST_FOR_SUMMARY: ${{ secrets.TEST_FOR_SUMMARY }}
+          DOCKER_INSTALL_TYPE: ${{ matrix.docker_install_type }}
+          DOCKER_INSTALL_VERSION: ${{ matrix.docker_install_version }}
       -
         name: Check coverage
         run: |

__tests__/docker/install.test.itg.ts

@@ -14,39 +14,20 @@
  * limitations under the License.
  */
 
-import {jest, describe, test, beforeEach, afterEach, expect} from '@jest/globals';
+import {describe, test, expect} from '@jest/globals';
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
 
-import {Install, InstallSourceArchive, InstallSourceImage} from '../../src/docker/install';
+import {Install, InstallSource, InstallSourceArchive, InstallSourceImage} from '../../src/docker/install';
 import {Docker} from '../../src/docker/docker';
 import {Exec} from '../../src/exec';
 
 const tmpDir = () => fs.mkdtempSync(path.join(process.env.TEMP || os.tmpdir(), 'docker-install-itg-'));
 
-describe('install', () => {
-  const originalEnv = process.env;
-  beforeEach(() => {
-    jest.resetModules();
-    process.env = {
-      ...originalEnv,
-      LIMA_START_ARGS: '--cpus 4 --memory 8',
-      LIMA_IMAGES: `x86_64:https://cloud.debian.org/images/cloud/bookworm/20231013-1532/debian-12-genericcloud-amd64-20231013-1532.qcow2@sha512:6b55e88b027c14da1b55c85a25a9f7069d4560a8fdb2d948c986a585db469728a06d2c528303e34bb62d8b2984def38fd9ddfc00965846ff6e05b01d6e883bfe
-aarch64:https://cloud.debian.org/images/cloud/bookworm/20231013-1532/debian-12-genericcloud-arm64-20231013-1532.qcow2`
-    };
-  });
-  afterEach(() => {
-    process.env = originalEnv;
-  });
+describe('root', () => {
   // prettier-ignore
-  test.each([
-    {type: 'image', tag: '27.3.1'} as InstallSourceImage,
-    {type: 'image', tag: 'master'} as InstallSourceImage,
-    {type: 'image', tag: 'latest'} as InstallSourceImage,
-    {type: 'archive', version: 'v26.1.4', channel: 'stable'} as InstallSourceArchive,
-    {type: 'archive', version: 'latest', channel: 'stable'} as InstallSourceArchive,
-  ])(
+  test.each(getSources(true))(
     'install docker %s', async (source) => {
       await ensureNoSystemContainerd();
       const install = new Install({
@@ -61,16 +42,12 @@ aarch64:https://cloud.debian.org/images/cloud/bookworm/20231013-1532/debian-12-g
 
 describe('rootless', () => {
   // prettier-ignore
-  test.each([
-    {type: 'image', tag: 'latest'} as InstallSourceImage,
-    {type: 'archive', version: 'latest', channel: 'stable'} as InstallSourceArchive,
-  ])(
+  test.each(getSources(false))(
     'install %s', async (source) => {
       // Skip on non linux
       if (os.platform() !== 'linux') {
         return;
       }
-
       await ensureNoSystemContainerd();
       const install = new Install({
         source: source,
@@ -122,3 +99,37 @@ async function ensureNoSystemContainerd() {
     });
   }
 }
+
+function getSources(root: boolean): Array<InstallSource> {
+  const dockerInstallType = process.env.DOCKER_INSTALL_TYPE;
+  const dockerInstallVersion = process.env.DOCKER_INSTALL_VERSION;
+  if (dockerInstallType && dockerInstallVersion) {
+    if (dockerInstallType === 'archive') {
+      // prettier-ignore
+      return [
+        { type: dockerInstallType, version: dockerInstallVersion, channel: 'stable'} as InstallSourceArchive
+      ];
+    } else {
+      // prettier-ignore
+      return [
+        { type: dockerInstallType, tag: dockerInstallVersion} as InstallSourceImage
+      ];
+    }
+  }
+  if (root) {
+    // prettier-ignore
+    return [
+      {type: 'image', tag: '27.3.1'} as InstallSourceImage,
+      {type: 'image', tag: 'master'} as InstallSourceImage,
+      {type: 'image', tag: 'latest'} as InstallSourceImage,
+      {type: 'archive', version: 'v26.1.4', channel: 'stable'} as InstallSourceArchive,
+      {type: 'archive', version: 'latest', channel: 'stable'} as InstallSourceArchive
+    ];
+  } else {
+    // prettier-ignore
+    return [
+      {type: 'image', tag: 'latest'} as InstallSourceImage,
+      {type: 'archive', version: 'latest', channel: 'stable'} as InstallSourceArchive
+    ];
+  }
+}

docker-bake.hcl

@@ -12,6 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+variable "NODE_VERSION" {
+  default = null
+}
+
 group "default" {
   targets = ["build"]
 }
@@ -24,31 +28,42 @@ group "validate" {
   targets = ["lint", "vendor-validate", "dockerfile-validate", "license-validate"]
 }
 
+target "_common" {
+  args = {
+    NODE_VERSION = NODE_VERSION
+  }
+}
+
 target "build" {
+  inherits = ["_common"]
   dockerfile = "dev.Dockerfile"
   target = "build-update"
   output = ["."]
 }
 
 target "format" {
+  inherits = ["_common"]
   dockerfile = "dev.Dockerfile"
   target = "format-update"
   output = ["."]
 }
 
 target "lint" {
+  inherits = ["_common"]
   dockerfile = "dev.Dockerfile"
   target = "lint"
   output = ["type=cacheonly"]
 }
 
 target "vendor" {
+  inherits = ["_common"]
   dockerfile = "dev.Dockerfile"
   target = "vendor-update"
   output = ["."]
 }
 
 target "vendor-validate" {
+  inherits = ["_common"]
   dockerfile = "dev.Dockerfile"
   target = "vendor-validate"
   output = ["type=cacheonly"]
@@ -67,6 +82,7 @@ target "dockerfile-validate" {
 }
 
 target "test" {
+  inherits = ["_common"]
   dockerfile = "dev.Dockerfile"
   target = "test"
   output = ["type=cacheonly"]
@@ -74,6 +90,7 @@ target "test" {
 }
 
 target "test-coverage" {
+  inherits = ["_common"]
   dockerfile = "dev.Dockerfile"
   target = "test-coverage"
   output = ["./coverage"]
@@ -87,6 +104,7 @@ variable "GITHUB_REF" {
 }
 
 target "publish" {
+  inherits = ["_common"]
   dockerfile = "dev.Dockerfile"
   args = {
     GITHUB_REF = GITHUB_REF

package.json

@@ -57,7 +57,7 @@
     "@octokit/core": "^5.1.0",
     "@octokit/plugin-rest-endpoint-methods": "^10.4.0",
     "async-retry": "^1.3.3",
-    "csv-parse": "^5.5.6",
+    "csv-parse": "^5.6.0",
     "gunzip-maybe": "^1.4.2",
     "handlebars": "^4.7.8",
     "he": "^1.2.0",

src/docker/assets.ts

@@ -132,6 +132,8 @@ Get-WinEvent -ea SilentlyContinue \`
 `;
 
 export const limaYamlData = `
+# Source: https://github.com/lima-vm/lima/blob/master/templates/docker-rootful.yaml
+
 # VM type: "qemu" or "vz" (on macOS 13 and later).
 # The vmType can be specified only on creating the instance.
 # The vmType of existing instances cannot be changed.
@@ -152,12 +154,18 @@ images:
   arch: "{{arch}}"
   digest: "{{digest}}"
 {{/each}}
-- location: "https://cloud-images.ubuntu.com/releases/22.04/release-20231026/ubuntu-22.04-server-cloudimg-amd64.img"
+- location: "https://cloud-images.ubuntu.com/releases/24.04/release-20241004/ubuntu-24.04-server-cloudimg-amd64.img"
   arch: "x86_64"
-  digest: "sha256:054db2d88c454bb0ad8dfd8883955e3946b57d2b0bf0d023f3ade3c93cdd14e5"
-- location: "https://cloud-images.ubuntu.com/releases/22.04/release-20231026/ubuntu-22.04-server-cloudimg-arm64.img"
+  digest: "sha256:fad101d50b06b26590cf30542349f9e9d3041ad7929e3bc3531c81ec27f2c788"
+- location: "https://cloud-images.ubuntu.com/releases/24.04/release-20241004/ubuntu-24.04-server-cloudimg-arm64.img"
+  arch: "aarch64"
+  digest: "sha256:e380b683b0c497d2a87af8a5dbe94c42eb54548fa976167f307ed8cf3944ec57"
+# Fallback to the latest release image.
+# Hint: run \`limactl prune\` to invalidate the cache
+- location: "https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img"
+  arch: "x86_64"
+- location: "https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img"
   arch: "aarch64"
-  digest: "sha256:eafa7742ce5ff109222ea313d31ea366d587b4e89b900b11d8285ae775dfe8c3"
 
 # CPUs
 # Builtin default: min(4, host CPU cores)

yarn.lock

@@ -1129,7 +1129,7 @@ __metadata:
     "@typescript-eslint/eslint-plugin": ^7.8.0
     "@typescript-eslint/parser": ^7.8.0
     async-retry: ^1.3.3
-    csv-parse: ^5.5.6
+    csv-parse: ^5.6.0
     dotenv: ^16.4.5
     eslint: ^8.57.0
     eslint-config-prettier: ^9.1.0
@@ -3420,10 +3420,10 @@ __metadata:
   languageName: node
   linkType: hard
 
-"csv-parse@npm:^5.5.6":
-  version: 5.5.6
-  resolution: "csv-parse@npm:5.5.6"
-  checksum: ee06f97f674487dc1d001b360de8ea510a41b9d971abf43bcf9c3be22c83a3634df0d3ebfbe52fd49d145077066be7ff9f25de3fc6b71aefb973099b04147a25
+"csv-parse@npm:^5.6.0":
+  version: 5.6.0
+  resolution: "csv-parse@npm:5.6.0"
+  checksum: 173e176bdaf212bab37d0f6d39a06d039d24a1c0ee40b9f1023ebf8b36095934807deeb493c0fb58592b39b0682ccd0be5c9e8d2b137c08807e7031595ea7a51
   languageName: node
   linkType: hard