From 964381b7e9ee7636379b0ba17c548775242ecdd4 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sat, 25 Feb 2023 09:40:22 +0100 Subject: [PATCH 01/12] move docker under dedicated folder Signed-off-by: CrazyMax --- __tests__/{ => docker}/docker.test.ts | 4 ++-- src/buildx/buildx.ts | 2 +- src/buildx/install.ts | 2 +- src/{ => docker}/docker.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename __tests__/{ => docker}/docker.test.ts (96%) rename src/{ => docker}/docker.ts (98%) diff --git a/__tests__/docker.test.ts b/__tests__/docker/docker.test.ts similarity index 96% rename from __tests__/docker.test.ts rename to __tests__/docker/docker.test.ts index 1907632..e32c64d 100644 --- a/__tests__/docker.test.ts +++ b/__tests__/docker/docker.test.ts @@ -19,8 +19,8 @@ import path from 'path'; import * as io from '@actions/io'; import osm = require('os'); -import {Docker} from '../src/docker'; -import {Exec} from '../src/exec'; +import {Docker} from '../../src/docker/docker'; +import {Exec} from '../../src/exec'; beforeEach(() => { jest.clearAllMocks(); diff --git a/src/buildx/buildx.ts b/src/buildx/buildx.ts index c59b89e..12f4c7a 100644 --- a/src/buildx/buildx.ts +++ b/src/buildx/buildx.ts @@ -19,7 +19,7 @@ import path from 'path'; import * as core from '@actions/core'; import * as semver from 'semver'; -import {Docker} from '../docker'; +import {Docker} from '../docker/docker'; import {Exec} from '../exec'; import {Inputs} from './inputs'; diff --git a/src/buildx/install.ts b/src/buildx/install.ts index 30425ab..5e2bdb9 100644 --- a/src/buildx/install.ts +++ b/src/buildx/install.ts @@ -26,7 +26,7 @@ import * as util from 'util'; import {Buildx} from './buildx'; import {Context} from '../context'; import {Exec} from '../exec'; -import {Docker} from '../docker'; +import {Docker} from '../docker/docker'; import {Git} from '../git'; import {GitHubRelease} from '../types/github'; diff --git a/src/docker.ts b/src/docker/docker.ts similarity index 98% rename from src/docker.ts rename to src/docker/docker.ts index 2ec3b33..9d391f3 100644 --- a/src/docker.ts +++ b/src/docker/docker.ts @@ -18,7 +18,7 @@ import os from 'os'; import path from 'path'; import * as core from '@actions/core'; import * as io from '@actions/io'; -import {Exec} from './exec'; +import {Exec} from '../exec'; export class Docker { static get configDir(): string { From 4d66b2fa08597fd4d456ea5b0030c34deb39d56d Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Mon, 27 Feb 2023 09:10:57 +0100 Subject: [PATCH 02/12] docker: install and download methods for macos and windows Signed-off-by: CrazyMax --- .github/workflows/e2e.yml | 39 ++++++ README.md | 1 + __tests__/docker/install.test.e2e.ts | 34 +++++ __tests__/docker/install.test.ts | 50 +++++++ jest.config.e2e.ts | 31 +++++ package.json | 6 +- scripts/setup-docker.ps1 | 69 ++++++++++ src/docker/install.ts | 195 +++++++++++++++++++++++++++ src/scripts.ts | 21 +++ src/util.ts | 24 ++++ tsconfig.json | 2 +- 11 files changed, 469 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/e2e.yml create mode 100644 __tests__/docker/install.test.e2e.ts create mode 100644 __tests__/docker/install.test.ts create mode 100644 jest.config.e2e.ts create mode 100644 scripts/setup-docker.ps1 create mode 100644 src/docker/install.ts create mode 100644 src/scripts.ts diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..b970040 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,39 @@ +name: e2e + +on: + workflow_dispatch: + push: + branches: + - 'main' + pull_request: + paths-ignore: + - '.github/buildx-releases.json' + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + #- ubuntu-latest + - macos-latest + - windows-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: 16 + cache: 'yarn' + - + name: Install + run: yarn install + - + name: Test + run: yarn test:e2e + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index c3e8a55..04b5ea0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ [![Version](https://img.shields.io/npm/v/@docker/actions-toolkit?label=version&logo=npm&style=flat-square)](https://www.npmjs.com/package/@docker/actions-toolkit) [![Downloads](https://img.shields.io/npm/dw/@docker/actions-toolkit?logo=npm&style=flat-square)](https://www.npmjs.com/package/@docker/actions-toolkit) [![Test workflow](https://img.shields.io/github/actions/workflow/status/docker/actions-toolkit/test.yml?label=test&logo=github&style=flat-square)](https://github.com/docker/actions-toolkit/actions?workflow=test) +[![E2E workflow](https://img.shields.io/github/actions/workflow/status/docker/actions-toolkit/e2e.yml?label=e2e&logo=github&style=flat-square)](https://github.com/docker/actions-toolkit/actions?workflow=e2e) [![Codecov](https://img.shields.io/codecov/c/github/docker/actions-toolkit?logo=codecov&style=flat-square)](https://codecov.io/gh/docker/actions-toolkit) # Actions Toolkit diff --git a/__tests__/docker/install.test.e2e.ts b/__tests__/docker/install.test.e2e.ts new file mode 100644 index 0000000..9b55615 --- /dev/null +++ b/__tests__/docker/install.test.e2e.ts @@ -0,0 +1,34 @@ +/** + * Copyright 2023 actions-toolkit authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {describe, expect, test} from '@jest/globals'; + +import {Install} from '../../src/docker/install'; +import {Docker} from '../../src/docker/docker'; + +describe('install', () => { + // prettier-ignore + test.each(['23.0.0'])( + 'install docker %s', async (version) => { + await expect((async () => { + const install = new Install(); + const toolPath = await install.download(version); + await install.install(toolPath); + await Docker.printVersion(); + await Docker.printInfo(); + })()).resolves.not.toThrow(); + }); +}); diff --git a/__tests__/docker/install.test.ts b/__tests__/docker/install.test.ts new file mode 100644 index 0000000..33f5415 --- /dev/null +++ b/__tests__/docker/install.test.ts @@ -0,0 +1,50 @@ +/** + * Copyright 2023 actions-toolkit authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {describe, expect, jest, test, beforeEach, afterEach} from '@jest/globals'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as rimraf from 'rimraf'; +import osm = require('os'); + +import {Install} from '../../src/docker/install'; + +// prettier-ignore +const tmpDir = path.join(process.env.TEMP || '/tmp', 'buildx-jest'); + +beforeEach(() => { + jest.clearAllMocks(); +}); + +afterEach(function () { + rimraf.sync(tmpDir); +}); + +describe('download', () => { + // prettier-ignore + test.each([ + ['19.03.6', 'linux'], + ['20.10.22', 'linux'], + ['20.10.22', 'darwin'], + ['20.10.22', 'win32'], + ])( + 'acquires %p of docker (%s)', async (version, platformOS) => { + jest.spyOn(osm, 'platform').mockImplementation(() => platformOS); + const install = new Install(); + const toolPath = await install.download(version); + expect(fs.existsSync(toolPath)).toBe(true); + }, 100000); +}); diff --git a/jest.config.e2e.ts b/jest.config.e2e.ts new file mode 100644 index 0000000..d2656db --- /dev/null +++ b/jest.config.e2e.ts @@ -0,0 +1,31 @@ +/** + * Copyright 2023 actions-toolkit authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module.exports = { + clearMocks: true, + testEnvironment: 'node', + moduleFileExtensions: ['js', 'ts'], + setupFiles: ['dotenv/config'], + testMatch: ['**/*.test.e2e.ts'], + testTimeout: 1800000, // 30 minutes + transform: { + '^.+\\.ts$': 'ts-jest' + }, + moduleNameMapper: { + '^csv-parse/sync': '/node_modules/csv-parse/dist/cjs/sync.cjs' + }, + verbose: false +}; diff --git a/package.json b/package.json index ab40ec6..19386bb 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "prettier": "prettier --check \"./**/*.ts\"", "prettier:fix": "prettier --write \"./**/*.ts\"", "test": "jest", - "test-coverage": "jest --coverage" + "test-coverage": "jest --coverage", + "test:e2e": "jest -c jest.config.e2e.ts" }, "repository": { "type": "git", @@ -35,7 +36,8 @@ "test": "__tests__" }, "files": [ - "lib" + "lib", + "scripts" ], "publishConfig": { "access": "public", diff --git a/scripts/setup-docker.ps1 b/scripts/setup-docker.ps1 new file mode 100644 index 0000000..982a003 --- /dev/null +++ b/scripts/setup-docker.ps1 @@ -0,0 +1,69 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$ToolDir, + + [Parameter(Mandatory = $false)] + [string]$TmpDir, + + [Parameter(Mandatory = $true)] + [string]$DockerHost) + +$pwver = (Get-ItemProperty -Path HKLM:\SOFTWARE\Microsoft\PowerShell\3\PowerShellEngine -Name 'PowerShellVersion').PowerShellVersion +Write-Host "PowerShell version: $pwver" + +# Create temp directory +if (!$TmpDir) { $TmpDir = $env:TEMP } +New-Item -ItemType Directory "$TmpDir" -ErrorAction SilentlyContinue | Out-Null + +# Remove existing service +if (Get-Service docker -ErrorAction SilentlyContinue) { + $dockerVersion = (docker version -f "{{.Server.Version}}") + Write-Host "Current installed Docker version: $dockerVersion" + # stop service + Stop-Service -Force -Name docker + Write-Host "Service stopped" + # remove service + sc.exe delete "docker" + # removes event log entry. we could use "Remove-EventLog -LogName -Source docker" + # but this cmd is not available atm + $ErrorActionPreference = "SilentlyContinue" + & reg delete "HKLM\SYSTEM\CurrentControlSet\Services\EventLog\Application\docker" /f 2>&1 | Out-Null + $ErrorActionPreference = "Stop" + Write-Host "Service removed" +} + +$env:DOCKER_HOST = $DockerHost +Write-Host "DOCKER_HOST: $env:DOCKER_HOST" + +Write-Host "Creating service" +New-Item -ItemType Directory "$TmpDir\moby-root" -ErrorAction SilentlyContinue | Out-Null +New-Item -ItemType Directory "$TmpDir\moby-exec" -ErrorAction SilentlyContinue | Out-Null +Start-Process -Wait -NoNewWindow "$ToolDir\dockerd" ` + -ArgumentList ` + "--host=$DockerHost", ` + "--data-root=$TmpDir\moby-root", ` + "--exec-root=$TmpDir\moby-exec", ` + "--pidfile=$TmpDir\docker.pid", ` + "--register-service" +Write-Host "Starting service" +Start-Service -Name docker +Write-Host "Service started successfully!" + +$tries=20 +Write-Host "Waiting for Docker daemon to start..." +While ($true) { + $ErrorActionPreference = "SilentlyContinue" + & "$ToolDir\docker" version | Out-Null + $ErrorActionPreference = "Stop" + If ($LastExitCode -eq 0) { + break + } + $tries-- + If ($tries -le 0) { + Throw "Failed to get a response from Docker daemon" + } + Write-Host -NoNewline "." + Start-Sleep -Seconds 1 +} +Write-Host "Docker daemon started successfully!" diff --git a/src/docker/install.ts b/src/docker/install.ts new file mode 100644 index 0000000..a3d6507 --- /dev/null +++ b/src/docker/install.ts @@ -0,0 +1,195 @@ +/** + * Copyright 2023 actions-toolkit authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import * as core from '@actions/core'; +import * as io from '@actions/io'; +import * as tc from '@actions/tool-cache'; +import * as util from 'util'; + +import * as scripts from '../scripts'; +import {Context} from '../context'; +import {Exec} from '../exec'; +import {Util} from '../util'; + +export class Install { + public async download(version: string, channel?: string): Promise { + channel = channel || 'stable'; + const downloadURL = this.downloadURL(version, channel); + + core.info(`Downloading ${downloadURL}`); + const downloadPath = await tc.downloadTool(downloadURL); + core.debug(`docker.Install.download downloadPath: ${downloadPath}`); + + let extractFolder: string; + if (os.platform() == 'win32') { + extractFolder = await tc.extractZip(downloadPath); + } else { + extractFolder = await tc.extractTar(downloadPath, path.join(Context.tmpDir(), 'docker')); + } + if (Util.isDirectory(path.join(extractFolder, 'docker'))) { + extractFolder = path.join(extractFolder, 'docker'); + } + core.debug(`docker.Install.download extractFolder: ${extractFolder}`); + + core.info('Fixing perms'); + fs.readdir(path.join(extractFolder), function (err, files) { + if (err) { + throw err; + } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + files.forEach(function (file, index) { + fs.chmodSync(path.join(extractFolder, file), '0755'); + }); + }); + + const tooldir = await tc.cacheDir(extractFolder, `docker-${channel}`, version.replace(/(0+)([1-9]+)/, '$2')); + core.addPath(tooldir); + core.info('Added Docker to PATH'); + return tooldir; + } + + public async install(toolDir: string): Promise { + switch (os.platform()) { + case 'darwin': { + await this.installDarwin(toolDir); + break; + } + case 'linux': { + await this.installLinux(toolDir); + break; + } + case 'win32': { + await this.installWindows(toolDir); + break; + } + default: { + throw new Error(`Unsupported platform: ${os.platform()}`); + } + } + } + + private async installDarwin(toolDir: string): Promise { + if (!(await Install.colimaInstalled())) { + await core.group('Installing colima', async () => { + await Exec.exec('brew', ['install', 'colima']); + }); + } + // colima is already started on the runner so env var added in download + // method is not expanded to the running process. + const envs = Object.assign({}, process.env, { + PATH: `${toolDir}:${process.env.PATH}` + }) as { + [key: string]: string; + }; + await core.group('Starting colima', async () => { + await Exec.exec('colima', ['start', '--runtime', 'docker', '--mount-type', '9p'], {env: envs}); + }); + } + + private async installLinux(toolDir: string): Promise { + core.addPath(toolDir); + core.info('Added Docker to PATH'); + } + + private async installWindows(toolDir: string): Promise { + const dockerHost = 'npipe:////./pipe/setup_docker_action'; + const setupCmd = await Util.powershellCommand(scripts.setupDockerPowershell, { + ToolDir: toolDir, + TmpDir: Context.tmpDir(), + DockerHost: dockerHost + }); + await core.group('Install Docker daemon service', async () => { + await Exec.exec(setupCmd.command, setupCmd.args); + }); + await core.group('Create Docker context', async () => { + await Exec.exec('docker', ['context', 'create', 'setup-docker-action', '--docker', `host=${dockerHost}`]); + await Exec.exec('docker', ['context', 'use', 'setup-docker-action']); + }); + } + + private downloadURL(version: string, channel: string): string { + let platformOS, platformArch: string; + switch (os.platform()) { + case 'darwin': { + platformOS = 'mac'; + break; + } + case 'linux': { + platformOS = 'linux'; + break; + } + case 'win32': { + platformOS = 'win'; + break; + } + default: { + platformOS = os.platform(); + break; + } + } + switch (os.arch()) { + case 'x64': { + platformArch = 'x86_64'; + break; + } + case 'ppc64': { + platformArch = 'ppc64le'; + break; + } + case 'arm': { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const arm_version = (process.config.variables as any).arm_version; + switch (arm_version) { + case 6: { + platformArch = 'armel'; + break; + } + case 7: { + platformArch = 'armhf'; + break; + } + default: { + platformArch = `v${arm_version}`; + break; + } + } + break; + } + default: { + platformArch = os.arch(); + break; + } + } + const ext = platformOS === 'win' ? '.zip' : '.tgz'; + return util.format('https://download.docker.com/%s/static/%s/%s/docker-%s%s', platformOS, channel, platformArch, version, ext); + } + + private static async colimaInstalled(): Promise { + return await io + .which('colima', true) + .then(res => { + core.debug(`docker.Install.colimaAvailable ok: ${res}`); + return true; + }) + .catch(error => { + core.debug(`docker.Install.colimaAvailable error: ${error}`); + return false; + }); + } +} diff --git a/src/scripts.ts b/src/scripts.ts new file mode 100644 index 0000000..0056f5a --- /dev/null +++ b/src/scripts.ts @@ -0,0 +1,21 @@ +/** + * Copyright 2023 actions-toolkit authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import path from 'path'; + +const scriptsPath = path.join(__dirname, '..', 'scripts'); + +export const setupDockerPowershell = path.join(scriptsPath, 'setup-docker.ps1'); diff --git a/src/util.ts b/src/util.ts index 2250139..bed6ee5 100644 --- a/src/util.ts +++ b/src/util.ts @@ -14,7 +14,9 @@ * limitations under the License. */ +import fs from 'fs'; import * as core from '@actions/core'; +import * as io from '@actions/io'; import {parse} from 'csv-parse/sync'; export interface InputListOpts { @@ -71,4 +73,26 @@ export class Util { } return true; } + + public static async powershellCommand(script: string, params: Record) { + const powershellPath: string = await io.which('powershell', true); + const escapedScript = script.replace(/'/g, "''").replace(/"|\n|\r/g, ''); + const escapedParams: string[] = []; + for (const key in params) { + escapedParams.push(`-${key} '${params[key].replace(/'/g, "''").replace(/"|\n|\r/g, '')}'`); + } + return { + command: `"${powershellPath}"`, + args: ['-NoLogo', '-Sta', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Unrestricted', '-Command', `& '${escapedScript}' ${escapedParams.join(' ')}`] + }; + } + + public static isDirectory(p) { + try { + return fs.lstatSync(p).isDirectory(); + } catch (_) { + // noop + } + return false; + } } diff --git a/tsconfig.json b/tsconfig.json index f6a43ac..9be7112 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -19,6 +19,6 @@ "./__tests__/**/*", "./lib/**/*", "node_modules", - "jest.config.ts" + "jest.config*.ts" ] } From 3ec6f00f46f6c9e2d0ac82829f67faa659ebede7 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Mon, 27 Feb 2023 09:59:45 +0100 Subject: [PATCH 03/12] docker: custom colima cfg on install Signed-off-by: CrazyMax --- __tests__/docker/install.test.e2e.ts | 2 +- package.json | 1 + scripts/colima.yaml | 175 +++++++++++++++++++++++++++ src/docker/install.ts | 84 ++++++++----- src/scripts.ts | 1 + yarn.lock | 49 ++++++++ 6 files changed, 282 insertions(+), 30 deletions(-) create mode 100644 scripts/colima.yaml diff --git a/__tests__/docker/install.test.e2e.ts b/__tests__/docker/install.test.e2e.ts index 9b55615..e75d60f 100644 --- a/__tests__/docker/install.test.e2e.ts +++ b/__tests__/docker/install.test.e2e.ts @@ -26,7 +26,7 @@ describe('install', () => { await expect((async () => { const install = new Install(); const toolPath = await install.download(version); - await install.install(toolPath); + await install.install(toolPath, version); await Docker.printVersion(); await Docker.printInfo(); })()).resolves.not.toThrow(); diff --git a/package.json b/package.json index 19386bb..f8aaf19 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "@actions/io": "^1.1.2", "@actions/tool-cache": "^2.0.1", "csv-parse": "^5.3.5", + "handlebars": "^4.7.7", "jwt-decode": "^3.1.2", "semver": "^7.3.8", "tmp": "^0.2.1" diff --git a/scripts/colima.yaml b/scripts/colima.yaml new file mode 100644 index 0000000..1c30642 --- /dev/null +++ b/scripts/colima.yaml @@ -0,0 +1,175 @@ +# Number of CPUs to be allocated to the virtual machine. +# Default: 2 +cpu: 2 + +# Size of the disk in GiB to be allocated to the virtual machine. +# NOTE: changing this has no effect after the virtual machine has been created. +# Default: 60 +disk: 60 + +# Size of the memory in GiB to be allocated to the virtual machine. +# Default: 2 +memory: 2 + +# Architecture of the virtual machine (x86_64, aarch64, host). +# Default: host +arch: host + +# Container runtime to be used (docker, containerd). +# Default: docker +runtime: docker + +# Kubernetes configuration for the virtual machine. +kubernetes: + enabled: false + +# Auto-activate on the Host for client access. +# Setting to true does the following on startup +# - sets as active Docker context (for Docker runtime). +# - sets as active Kubernetes context (if Kubernetes is enabled). +# Default: true +autoActivate: false + +# Network configurations for the virtual machine. +network: + # Assign reachable IP address to the virtual machine. + # NOTE: this is currently macOS only and ignored on Linux. + # Default: false + address: false + + # Custom DNS resolvers for the virtual machine. + # + # EXAMPLE + # dns: [8.8.8.8, 1.1.1.1] + # + # Default: [] + dns: [] + + # DNS hostnames to resolve to custom targets using the internal resolver. + # This setting has no effect if a custom DNS resolver list is supplied above. + # It does not configure the /etc/hosts files of any machine or container. + # The value can be an IP address or another host. + # + # EXAMPLE + # dnsHosts: + # example.com: 1.2.3.4 + dnsHosts: + host.docker.internal: host.lima.internal + + # Network driver to use (slirp, gvproxy), (requires vmType `qemu`) + # - slirp is the default user mode networking provided by Qemu + # - gvproxy is an alternative to VPNKit based on gVisor https://github.com/containers/gvisor-tap-vsock + # Default: gvproxy + driver: gvproxy + +# Forward the host's SSH agent to the virtual machine. +# Default: false +forwardAgent: false + +# Docker daemon configuration that maps directly to daemon.json. +# https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file. +# NOTE: some settings may affect Colima's ability to start docker. e.g. `hosts`. +# +# EXAMPLE - disable buildkit +# docker: +# features: +# buildkit: false +# +# EXAMPLE - add insecure registries +# docker: +# insecure-registries: +# - myregistry.com:5000 +# - host.docker.internal:5000 +# +# Colima default behaviour: buildkit enabled +# Default: {} +docker: {} + +# Virtual Machine type (qemu, vz) +# NOTE: this is macOS 13 only. For Linux and macOS <13.0, qemu is always used. +# +# vz is macOS virtualization framework and requires macOS 13 +# +# Default: qemu +vmType: qemu + +# Volume mount driver for the virtual machine (virtiofs, 9p, sshfs). +# +# virtiofs is limited to macOS and vmType `vz`. It is the fastest of the options. +# +# 9p is the recommended and the most stable option for vmType `qemu`. +# +# sshfs is faster than 9p but the least reliable of the options (when there are lots +# of concurrent reads or writes). +# +# Default: virtiofs (for vz), sshfs (for qemu) +mountType: 9p + +# The CPU type for the virtual machine (requires vmType `qemu`). +# Options available for host emulation can be checked with: `qemu-system-$(arch) -cpu help`. +# Instructions are also supported by appending to the cpu type e.g. "qemu64,+ssse3". +# Default: host +cpuType: host + +# For a more general purpose virtual machine, Ubuntu container is optionally provided +# as a layer on the virtual machine. +# The underlying virtual machine is still accessible via `colima ssh --layer=false` or running `colima` in +# the Ubuntu session. +# +# Default: false +layer: false + +# Custom provision scripts for the virtual machine. +# Provisioning scripts are executed on startup and therefore needs to be idempotent. +# +# EXAMPLE - script exected as root +# provision: +# - mode: system +# script: apk add htop vim +# +# EXAMPLE - script exected as user +# provision: +# - mode: user +# script: | +# [ -f ~/.provision ] && exit 0; +# echo provisioning as $USER... +# touch ~/.provision +# +# Default: [] +provision: + - mode: system + script: | + mkdir -p /tmp/docker-bins + cd /tmp/docker-bins + wget -qO- "https://download.docker.com/linux/static/{{dockerChannel}}/{{hostArch}}/docker-{{dockerVersion}}.tgz" | tar xvz --strip 1 + mv -f /tmp/docker-bins/* /usr/bin/ + +# Modify ~/.ssh/config automatically to include a SSH config for the virtual machine. +# SSH config will still be generated in ~/.colima/ssh_config regardless. +# Default: true +sshConfig: false + +# Configure volume mounts for the virtual machine. +# Colima mounts user's home directory by default to provide a familiar +# user experience. +# +# EXAMPLE +# mounts: +# - location: ~/secrets +# writable: false +# - location: ~/projects +# writable: true +# +# Colima default behaviour: $HOME and /tmp/colima are mounted as writable. +# Default: [] +mounts: [] + +# Environment variables for the virtual machine. +# +# EXAMPLE +# env: +# KEY: value +# ANOTHER_KEY: another value +# +# Default: {} +env: {} diff --git a/src/docker/install.ts b/src/docker/install.ts index a3d6507..d722ae2 100644 --- a/src/docker/install.ts +++ b/src/docker/install.ts @@ -17,10 +17,11 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; +import * as handlebars from 'handlebars'; +import * as util from 'util'; import * as core from '@actions/core'; import * as io from '@actions/io'; import * as tc from '@actions/tool-cache'; -import * as util from 'util'; import * as scripts from '../scripts'; import {Context} from '../context'; @@ -64,10 +65,11 @@ export class Install { return tooldir; } - public async install(toolDir: string): Promise { + public async install(toolDir: string, version: string, channel?: string): Promise { + channel = channel || 'stable'; switch (os.platform()) { case 'darwin': { - await this.installDarwin(toolDir); + await this.installDarwin(toolDir, version, channel); break; } case 'linux': { @@ -84,12 +86,33 @@ export class Install { } } - private async installDarwin(toolDir: string): Promise { + private async installDarwin(toolDir: string, version: string, channel?: string): Promise { + const colimaDir = path.join(os.homedir(), '.colima', 'default'); + await io.mkdirP(colimaDir); + const dockerHost = `unix://${colimaDir}/docker.sock`; + if (!(await Install.colimaInstalled())) { await core.group('Installing colima', async () => { await Exec.exec('brew', ['install', 'colima']); }); } + + await core.group('Creating colima config', async () => { + const colimaCfg = handlebars.compile( + fs.readFileSync(scripts.colimaConfig, { + encoding: 'utf8', + flag: 'r' + }) + )({ + hostArch: Install.platformArch(), + dockerVersion: version, + dockerChannel: channel + }); + core.info(`Writing colima config to ${path.join(colimaDir, 'colima.yaml')}`); + fs.writeFileSync(path.join(colimaDir, 'colima.yaml'), colimaCfg); + core.info(colimaCfg); + }); + // colima is already started on the runner so env var added in download // method is not expanded to the running process. const envs = Object.assign({}, process.env, { @@ -98,7 +121,12 @@ export class Install { [key: string]: string; }; await core.group('Starting colima', async () => { - await Exec.exec('colima', ['start', '--runtime', 'docker', '--mount-type', '9p'], {env: envs}); + await Exec.exec('colima', ['start', '--very-verbose'], {env: envs}); + }); + + await core.group('Create Docker context', async () => { + await Exec.exec('docker', ['context', 'create', 'setup-docker-action', '--docker', `host=${dockerHost}`]); + await Exec.exec('docker', ['context', 'use', 'setup-docker-action']); }); } @@ -109,6 +137,7 @@ export class Install { private async installWindows(toolDir: string): Promise { const dockerHost = 'npipe:////./pipe/setup_docker_action'; + const setupCmd = await Util.powershellCommand(scripts.setupDockerPowershell, { ToolDir: toolDir, TmpDir: Context.tmpDir(), @@ -117,6 +146,7 @@ export class Install { await core.group('Install Docker daemon service', async () => { await Exec.exec(setupCmd.command, setupCmd.args); }); + await core.group('Create Docker context', async () => { await Exec.exec('docker', ['context', 'create', 'setup-docker-action', '--docker', `host=${dockerHost}`]); await Exec.exec('docker', ['context', 'use', 'setup-docker-action']); @@ -124,60 +154,56 @@ export class Install { } private downloadURL(version: string, channel: string): string { - let platformOS, platformArch: string; + const platformOS = Install.platformOS(); + const platformArch = Install.platformArch(); + const ext = platformOS === 'win' ? '.zip' : '.tgz'; + return util.format('https://download.docker.com/%s/static/%s/%s/docker-%s%s', platformOS, channel, platformArch, version, ext); + } + + private static platformOS(): string { switch (os.platform()) { case 'darwin': { - platformOS = 'mac'; - break; + return 'mac'; } case 'linux': { - platformOS = 'linux'; - break; + return 'linux'; } case 'win32': { - platformOS = 'win'; - break; + return 'win'; } default: { - platformOS = os.platform(); - break; + return os.platform(); } } + } + + private static platformArch(): string { switch (os.arch()) { case 'x64': { - platformArch = 'x86_64'; - break; + return 'x86_64'; } case 'ppc64': { - platformArch = 'ppc64le'; - break; + return 'ppc64le'; } case 'arm': { // eslint-disable-next-line @typescript-eslint/no-explicit-any const arm_version = (process.config.variables as any).arm_version; switch (arm_version) { case 6: { - platformArch = 'armel'; - break; + return 'armel'; } case 7: { - platformArch = 'armhf'; - break; + return 'armhf'; } default: { - platformArch = `v${arm_version}`; - break; + return `v${arm_version}`; } } - break; } default: { - platformArch = os.arch(); - break; + return os.arch(); } } - const ext = platformOS === 'win' ? '.zip' : '.tgz'; - return util.format('https://download.docker.com/%s/static/%s/%s/docker-%s%s', platformOS, channel, platformArch, version, ext); } private static async colimaInstalled(): Promise { diff --git a/src/scripts.ts b/src/scripts.ts index 0056f5a..ac33853 100644 --- a/src/scripts.ts +++ b/src/scripts.ts @@ -19,3 +19,4 @@ import path from 'path'; const scriptsPath = path.join(__dirname, '..', 'scripts'); export const setupDockerPowershell = path.join(scriptsPath, 'setup-docker.ps1'); +export const colimaConfig = path.join(scriptsPath, 'colima.yaml'); diff --git a/yarn.lock b/yarn.lock index 2e18697..5a5562a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -782,6 +782,7 @@ __metadata: eslint-plugin-import: ^2.27.5 eslint-plugin-jest: ^26.9.0 eslint-plugin-prettier: ^4.2.1 + handlebars: ^4.7.7 jest: ^27.5.1 jwt-decode: ^3.1.2 prettier: ^2.8.3 @@ -3667,6 +3668,24 @@ __metadata: languageName: node linkType: hard +"handlebars@npm:^4.7.7": + version: 4.7.7 + resolution: "handlebars@npm:4.7.7" + dependencies: + minimist: ^1.2.5 + neo-async: ^2.6.0 + source-map: ^0.6.1 + uglify-js: ^3.1.4 + wordwrap: ^1.0.0 + dependenciesMeta: + uglify-js: + optional: true + bin: + handlebars: bin/handlebars + checksum: 1e79a43f5e18d15742977cb987923eab3e2a8f44f2d9d340982bcb69e1735ed049226e534d7c1074eaddaf37e4fb4f471a8adb71cddd5bc8cf3f894241df5cee + languageName: node + linkType: hard + "hard-rejection@npm:^2.1.0": version: 2.1.0 resolution: "hard-rejection@npm:2.1.0" @@ -5144,6 +5163,13 @@ __metadata: languageName: node linkType: hard +"minimist@npm:^1.2.5": + version: 1.2.8 + resolution: "minimist@npm:1.2.8" + checksum: 75a6d645fb122dad29c06a7597bddea977258957ed88d7a6df59b5cd3fe4a527e253e9bbf2e783e4b73657f9098b96a5fe96ab8a113655d4109108577ecf85b0 + languageName: node + linkType: hard + "minipass-collect@npm:^1.0.2": version: 1.0.2 resolution: "minipass-collect@npm:1.0.2" @@ -5267,6 +5293,13 @@ __metadata: languageName: node linkType: hard +"neo-async@npm:^2.6.0": + version: 2.6.2 + resolution: "neo-async@npm:2.6.2" + checksum: deac9f8d00eda7b2e5cd1b2549e26e10a0faa70adaa6fdadca701cc55f49ee9018e427f424bac0c790b7c7e2d3068db97f3093f1093975f2acb8f8818b936ed9 + languageName: node + linkType: hard + "nested-error-stacks@npm:^2.0.0, nested-error-stacks@npm:^2.1.0": version: 2.1.1 resolution: "nested-error-stacks@npm:2.1.1" @@ -6694,6 +6727,15 @@ __metadata: languageName: node linkType: hard +"uglify-js@npm:^3.1.4": + version: 3.17.4 + resolution: "uglify-js@npm:3.17.4" + bin: + uglifyjs: bin/uglifyjs + checksum: 7b3897df38b6fc7d7d9f4dcd658599d81aa2b1fb0d074829dd4e5290f7318dbca1f4af2f45acb833b95b1fe0ed4698662ab61b87e94328eb4c0a0d3435baf924 + languageName: node + linkType: hard + "unbox-primitive@npm:^1.0.2": version: 1.0.2 resolution: "unbox-primitive@npm:1.0.2" @@ -6939,6 +6981,13 @@ __metadata: languageName: node linkType: hard +"wordwrap@npm:^1.0.0": + version: 1.0.0 + resolution: "wordwrap@npm:1.0.0" + checksum: 2a44b2788165d0a3de71fd517d4880a8e20ea3a82c080ce46e294f0b68b69a2e49cff5f99c600e275c698a90d12c5ea32aff06c311f0db2eb3f1201f3e7b2a04 + languageName: node + linkType: hard + "wrap-ansi@npm:^7.0.0": version: 7.0.0 resolution: "wrap-ansi@npm:7.0.0" From 93fa96f54fac0b04eb44a00de651f97e6faaae76 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 28 Feb 2023 20:05:08 +0100 Subject: [PATCH 04/12] docker: install for linux Signed-off-by: CrazyMax --- .github/workflows/e2e.yml | 2 +- package.json | 2 +- scripts/setup-docker-linux.sh | 51 +++++++++++++++++++ ...{setup-docker.ps1 => setup-docker-win.ps1} | 0 src/docker/install.ts | 23 +++++++-- src/scripts.ts | 3 +- 6 files changed, 75 insertions(+), 6 deletions(-) create mode 100755 scripts/setup-docker-linux.sh rename scripts/{setup-docker.ps1 => setup-docker-win.ps1} (100%) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b970040..024ae10 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -16,7 +16,7 @@ jobs: fail-fast: false matrix: os: - #- ubuntu-latest + - ubuntu-latest - macos-latest - windows-latest steps: diff --git a/package.json b/package.json index f8aaf19..6886fd1 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "prettier:fix": "prettier --write \"./**/*.ts\"", "test": "jest", "test-coverage": "jest --coverage", - "test:e2e": "jest -c jest.config.e2e.ts" + "test:e2e": "jest -c jest.config.e2e.ts --runInBand --detectOpenHandles --forceExit" }, "repository": { "type": "git", diff --git a/scripts/setup-docker-linux.sh b/scripts/setup-docker-linux.sh new file mode 100755 index 0000000..763a707 --- /dev/null +++ b/scripts/setup-docker-linux.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -eu + +: "${TOOLDIR=}" +: "${RUNDIR=}" +: "${DOCKER_HOST=}" + +export PATH="$TOOLDIR::$PATH" + +if [ -z "$DOCKER_HOST" ]; then + echo >&2 'error: DOCKER_HOST required' + false +fi + +if ! command -v dockerd &> /dev/null; then + echo >&2 'error: dockerd missing from PATH' + false +fi + +( + echo "Starting dockerd" + set -x + exec dockerd \ + --host="$DOCKER_HOST" \ + --exec-root="$RUNDIR/execroot" \ + --data-root="$RUNDIR/data" \ + --pidfile="$RUNDIR/docker.pid" \ + --userland-proxy=false \ + 2>&1 | tee "$RUNDIR/dockerd.log" +) & + +tries=60 +echo "Waiting for daemon to start..." +while ! docker version &> /dev/null; do + ((tries--)) + if [ $tries -le 0 ]; then + printf "\n" + if [ -z "$DOCKER_HOST" ]; then + echo >&2 "error: daemon failed to start" + echo >&2 " check $RUNDIR/docker.log for details" + else + echo >&2 "error: daemon at $DOCKER_HOST fails to 'docker version':" + docker version >&2 || true + fi + false + fi + printf "." + sleep 2 +done +printf "\n" diff --git a/scripts/setup-docker.ps1 b/scripts/setup-docker-win.ps1 similarity index 100% rename from scripts/setup-docker.ps1 rename to scripts/setup-docker-win.ps1 diff --git a/src/docker/install.ts b/src/docker/install.ts index d722ae2..687c07f 100644 --- a/src/docker/install.ts +++ b/src/docker/install.ts @@ -131,14 +131,31 @@ export class Install { } private async installLinux(toolDir: string): Promise { - core.addPath(toolDir); - core.info('Added Docker to PATH'); + const dockerHost = `unix://${path.join(Context.tmpDir(), 'docker.sock')}`; + + await core.group('Install Docker daemon', async () => { + const bashPath: string = await io.which('bash', true); + await Exec.exec('sudo', ['-E', bashPath, scripts.setupDockerLinux], { + env: Object.assign({}, process.env, { + TOOLDIR: toolDir, + RUNDIR: Context.tmpDir(), + DOCKER_HOST: dockerHost + }) as { + [key: string]: string; + } + }); + }); + + await core.group('Create Docker context', async () => { + await Exec.exec('docker', ['context', 'create', 'setup-docker-action', '--docker', `host=${dockerHost}`]); + await Exec.exec('docker', ['context', 'use', 'setup-docker-action']); + }); } private async installWindows(toolDir: string): Promise { const dockerHost = 'npipe:////./pipe/setup_docker_action'; - const setupCmd = await Util.powershellCommand(scripts.setupDockerPowershell, { + const setupCmd = await Util.powershellCommand(scripts.setupDockerWin, { ToolDir: toolDir, TmpDir: Context.tmpDir(), DockerHost: dockerHost diff --git a/src/scripts.ts b/src/scripts.ts index ac33853..600c1e3 100644 --- a/src/scripts.ts +++ b/src/scripts.ts @@ -18,5 +18,6 @@ import path from 'path'; const scriptsPath = path.join(__dirname, '..', 'scripts'); -export const setupDockerPowershell = path.join(scriptsPath, 'setup-docker.ps1'); +export const setupDockerLinux = path.join(scriptsPath, 'setup-docker-linux.sh'); +export const setupDockerWin = path.join(scriptsPath, 'setup-docker-win.ps1'); export const colimaConfig = path.join(scriptsPath, 'colima.yaml'); From 3b532d1b9160a990cc4aa3298e10818aefcf3e2e Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 28 Feb 2023 20:15:33 +0100 Subject: [PATCH 05/12] docker: install with custom runDir Signed-off-by: CrazyMax --- __tests__/docker/install.test.e2e.ts | 6 +++++- scripts/setup-docker-linux.sh | 29 +++++++++++++--------------- scripts/setup-docker-win.ps1 | 19 +++++++++--------- src/docker/install.ts | 21 ++++++++++---------- 4 files changed, 37 insertions(+), 38 deletions(-) diff --git a/__tests__/docker/install.test.e2e.ts b/__tests__/docker/install.test.e2e.ts index e75d60f..243f0a8 100644 --- a/__tests__/docker/install.test.e2e.ts +++ b/__tests__/docker/install.test.e2e.ts @@ -14,11 +14,15 @@ * limitations under the License. */ +import path from 'path'; import {describe, expect, test} from '@jest/globals'; import {Install} from '../../src/docker/install'; import {Docker} from '../../src/docker/docker'; +// prettier-ignore +const tmpDir = path.join(process.env.TEMP || '/tmp', 'buildx-jest'); + describe('install', () => { // prettier-ignore test.each(['23.0.0'])( @@ -26,7 +30,7 @@ describe('install', () => { await expect((async () => { const install = new Install(); const toolPath = await install.download(version); - await install.install(toolPath, version); + await install.install(toolPath, tmpDir, version); await Docker.printVersion(); await Docker.printInfo(); })()).resolves.not.toThrow(); diff --git a/scripts/setup-docker-linux.sh b/scripts/setup-docker-linux.sh index 763a707..239724f 100755 --- a/scripts/setup-docker-linux.sh +++ b/scripts/setup-docker-linux.sh @@ -18,6 +18,8 @@ if ! command -v dockerd &> /dev/null; then false fi +mkdir -p "$RUNDIR" + ( echo "Starting dockerd" set -x @@ -31,21 +33,16 @@ fi ) & tries=60 -echo "Waiting for daemon to start..." while ! docker version &> /dev/null; do - ((tries--)) - if [ $tries -le 0 ]; then - printf "\n" - if [ -z "$DOCKER_HOST" ]; then - echo >&2 "error: daemon failed to start" - echo >&2 " check $RUNDIR/docker.log for details" - else - echo >&2 "error: daemon at $DOCKER_HOST fails to 'docker version':" - docker version >&2 || true - fi - false - fi - printf "." - sleep 2 + ((tries--)) + if [ $tries -le 0 ]; then + if [ -z "$DOCKER_HOST" ]; then + echo >&2 "error: daemon failed to start" + else + echo >&2 "error: daemon at $DOCKER_HOST fails to 'docker version':" + docker version >&2 || true + fi + false + fi + sleep 2 done -printf "\n" diff --git a/scripts/setup-docker-win.ps1 b/scripts/setup-docker-win.ps1 index 982a003..7ba3e5d 100644 --- a/scripts/setup-docker-win.ps1 +++ b/scripts/setup-docker-win.ps1 @@ -3,8 +3,8 @@ param( [Parameter(Mandatory = $true)] [string]$ToolDir, - [Parameter(Mandatory = $false)] - [string]$TmpDir, + [Parameter(Mandatory = $true)] + [string]$RunDir, [Parameter(Mandatory = $true)] [string]$DockerHost) @@ -12,9 +12,8 @@ param( $pwver = (Get-ItemProperty -Path HKLM:\SOFTWARE\Microsoft\PowerShell\3\PowerShellEngine -Name 'PowerShellVersion').PowerShellVersion Write-Host "PowerShell version: $pwver" -# Create temp directory -if (!$TmpDir) { $TmpDir = $env:TEMP } -New-Item -ItemType Directory "$TmpDir" -ErrorAction SilentlyContinue | Out-Null +# Create run directory +New-Item -ItemType Directory "$RunDir" -ErrorAction SilentlyContinue | Out-Null # Remove existing service if (Get-Service docker -ErrorAction SilentlyContinue) { @@ -37,14 +36,14 @@ $env:DOCKER_HOST = $DockerHost Write-Host "DOCKER_HOST: $env:DOCKER_HOST" Write-Host "Creating service" -New-Item -ItemType Directory "$TmpDir\moby-root" -ErrorAction SilentlyContinue | Out-Null -New-Item -ItemType Directory "$TmpDir\moby-exec" -ErrorAction SilentlyContinue | Out-Null +New-Item -ItemType Directory "$RunDir\moby-root" -ErrorAction SilentlyContinue | Out-Null +New-Item -ItemType Directory "$RunDir\moby-exec" -ErrorAction SilentlyContinue | Out-Null Start-Process -Wait -NoNewWindow "$ToolDir\dockerd" ` -ArgumentList ` "--host=$DockerHost", ` - "--data-root=$TmpDir\moby-root", ` - "--exec-root=$TmpDir\moby-exec", ` - "--pidfile=$TmpDir\docker.pid", ` + "--data-root=$RunDir\moby-root", ` + "--exec-root=$RunDir\moby-exec", ` + "--pidfile=$RunDir\docker.pid", ` "--register-service" Write-Host "Starting service" Start-Service -Name docker diff --git a/src/docker/install.ts b/src/docker/install.ts index 687c07f..9b20435 100644 --- a/src/docker/install.ts +++ b/src/docker/install.ts @@ -24,7 +24,6 @@ import * as io from '@actions/io'; import * as tc from '@actions/tool-cache'; import * as scripts from '../scripts'; -import {Context} from '../context'; import {Exec} from '../exec'; import {Util} from '../util'; @@ -41,7 +40,7 @@ export class Install { if (os.platform() == 'win32') { extractFolder = await tc.extractZip(downloadPath); } else { - extractFolder = await tc.extractTar(downloadPath, path.join(Context.tmpDir(), 'docker')); + extractFolder = await tc.extractTar(downloadPath); } if (Util.isDirectory(path.join(extractFolder, 'docker'))) { extractFolder = path.join(extractFolder, 'docker'); @@ -65,7 +64,7 @@ export class Install { return tooldir; } - public async install(toolDir: string, version: string, channel?: string): Promise { + public async install(toolDir: string, runDir: string, version: string, channel?: string): Promise { channel = channel || 'stable'; switch (os.platform()) { case 'darwin': { @@ -73,11 +72,11 @@ export class Install { break; } case 'linux': { - await this.installLinux(toolDir); + await this.installLinux(toolDir, runDir); break; } case 'win32': { - await this.installWindows(toolDir); + await this.installWindows(toolDir, runDir); break; } default: { @@ -87,7 +86,7 @@ export class Install { } private async installDarwin(toolDir: string, version: string, channel?: string): Promise { - const colimaDir = path.join(os.homedir(), '.colima', 'default'); + const colimaDir = path.join(os.homedir(), '.colima', 'default'); // TODO: create a custom colima profile to avoid overlap with other actions await io.mkdirP(colimaDir); const dockerHost = `unix://${colimaDir}/docker.sock`; @@ -130,15 +129,15 @@ export class Install { }); } - private async installLinux(toolDir: string): Promise { - const dockerHost = `unix://${path.join(Context.tmpDir(), 'docker.sock')}`; + private async installLinux(toolDir: string, runDir: string): Promise { + const dockerHost = `unix://${path.join(runDir, 'docker.sock')}`; await core.group('Install Docker daemon', async () => { const bashPath: string = await io.which('bash', true); await Exec.exec('sudo', ['-E', bashPath, scripts.setupDockerLinux], { env: Object.assign({}, process.env, { TOOLDIR: toolDir, - RUNDIR: Context.tmpDir(), + RUNDIR: runDir, DOCKER_HOST: dockerHost }) as { [key: string]: string; @@ -152,12 +151,12 @@ export class Install { }); } - private async installWindows(toolDir: string): Promise { + private async installWindows(toolDir: string, runDir: string): Promise { const dockerHost = 'npipe:////./pipe/setup_docker_action'; const setupCmd = await Util.powershellCommand(scripts.setupDockerWin, { ToolDir: toolDir, - TmpDir: Context.tmpDir(), + RunDir: runDir, DockerHost: dockerHost }); await core.group('Install Docker daemon service', async () => { From 6210eb507a18450e993dcad42c468038c8105e36 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 28 Feb 2023 20:36:54 +0100 Subject: [PATCH 06/12] docker: print logs from windows service after install Signed-off-by: CrazyMax --- scripts/setup-docker-win.ps1 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/setup-docker-win.ps1 b/scripts/setup-docker-win.ps1 index 7ba3e5d..7fa1ecb 100644 --- a/scripts/setup-docker-win.ps1 +++ b/scripts/setup-docker-win.ps1 @@ -66,3 +66,8 @@ While ($true) { Start-Sleep -Seconds 1 } Write-Host "Docker daemon started successfully!" + +Get-WinEvent -ea SilentlyContinue ` + -FilterHashtable @{ProviderName= "docker"; LogName = "application"} | + Sort-Object @{Expression="TimeCreated";Descending=$false} | + ForEach-Object {"$($_.TimeCreated.ToUniversalTime().ToString("o")) [$($_.LevelDisplayName)] $($_.Message)"} From 2201fc119470c7ed1402d6103a640b4e2e644708 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 28 Feb 2023 20:55:32 +0100 Subject: [PATCH 07/12] e2e: coverage reports Signed-off-by: CrazyMax --- .github/workflows/e2e.yml | 8 +++++++- .github/workflows/test.yml | 1 + package.json | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 024ae10..0715f9d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -34,6 +34,12 @@ jobs: run: yarn install - name: Test - run: yarn test:e2e + run: yarn test-coverage:e2e --coverageDirectory=./coverage env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - + name: Upload coverage + uses: codecov/codecov-action@v3 + with: + file: ./coverage/clover.xml + flags: e2e diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 693bbb0..ad7ad35 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,3 +46,4 @@ jobs: uses: codecov/codecov-action@v3 with: file: ./coverage/clover.xml + flags: unit diff --git a/package.json b/package.json index 6886fd1..9a1f74b 100644 --- a/package.json +++ b/package.json @@ -11,8 +11,9 @@ "prettier": "prettier --check \"./**/*.ts\"", "prettier:fix": "prettier --write \"./**/*.ts\"", "test": "jest", + "test:e2e": "jest -c jest.config.e2e.ts --runInBand --detectOpenHandles --forceExit", "test-coverage": "jest --coverage", - "test:e2e": "jest -c jest.config.e2e.ts --runInBand --detectOpenHandles --forceExit" + "test-coverage:e2e": "jest --coverage -c jest.config.e2e.ts --runInBand --detectOpenHandles --forceExit" }, "repository": { "type": "git", From 3a0f4db686d1ce9756707834d03d82457a260afb Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 28 Feb 2023 21:17:50 +0100 Subject: [PATCH 08/12] fix license headers Signed-off-by: CrazyMax --- hack/dockerfiles/license.Dockerfile | 2 +- scripts/setup-docker-linux.sh | 14 ++++++++++++++ scripts/setup-docker-win.ps1 | 14 ++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/hack/dockerfiles/license.Dockerfile b/hack/dockerfiles/license.Dockerfile index 9127510..dc0434e 100644 --- a/hack/dockerfiles/license.Dockerfile +++ b/hack/dockerfiles/license.Dockerfile @@ -16,7 +16,7 @@ ARG LICENSE_HOLDER="actions-toolkit authors" ARG LICENSE_TYPE="apache" -ARG LICENSE_FILES=".*\(Dockerfile\|Makefile\|\.js\|\.ts\|\.hcl\|\.sh\)" +ARG LICENSE_FILES=".*\(Dockerfile\|Makefile\|\.js\|\.ts\|\.hcl\|\.sh|\.ps1\)" ARG ADDLICENSE_VERSION="v1.0.0" FROM ghcr.io/google/addlicense:${ADDLICENSE_VERSION} AS addlicense diff --git a/scripts/setup-docker-linux.sh b/scripts/setup-docker-linux.sh index 239724f..e09a765 100755 --- a/scripts/setup-docker-linux.sh +++ b/scripts/setup-docker-linux.sh @@ -1,5 +1,19 @@ #!/usr/bin/env bash +# Copyright 2023 actions-toolkit authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + set -eu : "${TOOLDIR=}" diff --git a/scripts/setup-docker-win.ps1 b/scripts/setup-docker-win.ps1 index 7fa1ecb..e4c28b9 100644 --- a/scripts/setup-docker-win.ps1 +++ b/scripts/setup-docker-win.ps1 @@ -1,3 +1,17 @@ +# Copyright 2023 actions-toolkit authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [CmdletBinding()] param( [Parameter(Mandatory = $true)] From adad1e3786aadd578bea36e57c5ca8af37a87ee9 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Tue, 28 Feb 2023 21:37:29 +0100 Subject: [PATCH 09/12] ci: build workflow Signed-off-by: CrazyMax --- .github/workflows/build.yml | 22 ++++++++++++++++++++++ README.md | 1 + 2 files changed, 23 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..c6369b9 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,22 @@ +name: build + +on: + push: + branches: + - 'main' + pull_request: + paths-ignore: + - '.github/buildx-releases.json' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Build + uses: docker/bake-action@v2 + with: + targets: build diff --git a/README.md b/README.md index 04b5ea0..dabd51e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Version](https://img.shields.io/npm/v/@docker/actions-toolkit?label=version&logo=npm&style=flat-square)](https://www.npmjs.com/package/@docker/actions-toolkit) [![Downloads](https://img.shields.io/npm/dw/@docker/actions-toolkit?logo=npm&style=flat-square)](https://www.npmjs.com/package/@docker/actions-toolkit) +[![Build workflow](https://img.shields.io/github/actions/workflow/status/docker/actions-toolkit/build.yml?label=build&logo=github&style=flat-square)](https://github.com/docker/actions-toolkit/actions?workflow=build) [![Test workflow](https://img.shields.io/github/actions/workflow/status/docker/actions-toolkit/test.yml?label=test&logo=github&style=flat-square)](https://github.com/docker/actions-toolkit/actions?workflow=test) [![E2E workflow](https://img.shields.io/github/actions/workflow/status/docker/actions-toolkit/e2e.yml?label=e2e&logo=github&style=flat-square)](https://github.com/docker/actions-toolkit/actions?workflow=e2e) [![Codecov](https://img.shields.io/codecov/c/github/docker/actions-toolkit?logo=codecov&style=flat-square)](https://codecov.io/gh/docker/actions-toolkit) From 3c2fe5ddb272bd87987b62b43afa0aa2f7256b56 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 2 Mar 2023 11:01:19 +0100 Subject: [PATCH 10/12] docker: move assets to ts Signed-off-by: CrazyMax --- package.json | 3 +- scripts/colima.yaml | 175 ----------------- scripts/setup-docker-linux.sh | 62 ------ scripts/setup-docker-win.ps1 | 87 --------- src/docker/assets.ts | 348 ++++++++++++++++++++++++++++++++++ src/docker/install.ts | 19 +- src/scripts.ts | 23 --- 7 files changed, 359 insertions(+), 358 deletions(-) delete mode 100644 scripts/colima.yaml delete mode 100755 scripts/setup-docker-linux.sh delete mode 100644 scripts/setup-docker-win.ps1 create mode 100644 src/docker/assets.ts delete mode 100644 src/scripts.ts diff --git a/package.json b/package.json index 9a1f74b..71f6ba7 100644 --- a/package.json +++ b/package.json @@ -37,8 +37,7 @@ "test": "__tests__" }, "files": [ - "lib", - "scripts" + "lib" ], "publishConfig": { "access": "public", diff --git a/scripts/colima.yaml b/scripts/colima.yaml deleted file mode 100644 index 1c30642..0000000 --- a/scripts/colima.yaml +++ /dev/null @@ -1,175 +0,0 @@ -# Number of CPUs to be allocated to the virtual machine. -# Default: 2 -cpu: 2 - -# Size of the disk in GiB to be allocated to the virtual machine. -# NOTE: changing this has no effect after the virtual machine has been created. -# Default: 60 -disk: 60 - -# Size of the memory in GiB to be allocated to the virtual machine. -# Default: 2 -memory: 2 - -# Architecture of the virtual machine (x86_64, aarch64, host). -# Default: host -arch: host - -# Container runtime to be used (docker, containerd). -# Default: docker -runtime: docker - -# Kubernetes configuration for the virtual machine. -kubernetes: - enabled: false - -# Auto-activate on the Host for client access. -# Setting to true does the following on startup -# - sets as active Docker context (for Docker runtime). -# - sets as active Kubernetes context (if Kubernetes is enabled). -# Default: true -autoActivate: false - -# Network configurations for the virtual machine. -network: - # Assign reachable IP address to the virtual machine. - # NOTE: this is currently macOS only and ignored on Linux. - # Default: false - address: false - - # Custom DNS resolvers for the virtual machine. - # - # EXAMPLE - # dns: [8.8.8.8, 1.1.1.1] - # - # Default: [] - dns: [] - - # DNS hostnames to resolve to custom targets using the internal resolver. - # This setting has no effect if a custom DNS resolver list is supplied above. - # It does not configure the /etc/hosts files of any machine or container. - # The value can be an IP address or another host. - # - # EXAMPLE - # dnsHosts: - # example.com: 1.2.3.4 - dnsHosts: - host.docker.internal: host.lima.internal - - # Network driver to use (slirp, gvproxy), (requires vmType `qemu`) - # - slirp is the default user mode networking provided by Qemu - # - gvproxy is an alternative to VPNKit based on gVisor https://github.com/containers/gvisor-tap-vsock - # Default: gvproxy - driver: gvproxy - -# Forward the host's SSH agent to the virtual machine. -# Default: false -forwardAgent: false - -# Docker daemon configuration that maps directly to daemon.json. -# https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file. -# NOTE: some settings may affect Colima's ability to start docker. e.g. `hosts`. -# -# EXAMPLE - disable buildkit -# docker: -# features: -# buildkit: false -# -# EXAMPLE - add insecure registries -# docker: -# insecure-registries: -# - myregistry.com:5000 -# - host.docker.internal:5000 -# -# Colima default behaviour: buildkit enabled -# Default: {} -docker: {} - -# Virtual Machine type (qemu, vz) -# NOTE: this is macOS 13 only. For Linux and macOS <13.0, qemu is always used. -# -# vz is macOS virtualization framework and requires macOS 13 -# -# Default: qemu -vmType: qemu - -# Volume mount driver for the virtual machine (virtiofs, 9p, sshfs). -# -# virtiofs is limited to macOS and vmType `vz`. It is the fastest of the options. -# -# 9p is the recommended and the most stable option for vmType `qemu`. -# -# sshfs is faster than 9p but the least reliable of the options (when there are lots -# of concurrent reads or writes). -# -# Default: virtiofs (for vz), sshfs (for qemu) -mountType: 9p - -# The CPU type for the virtual machine (requires vmType `qemu`). -# Options available for host emulation can be checked with: `qemu-system-$(arch) -cpu help`. -# Instructions are also supported by appending to the cpu type e.g. "qemu64,+ssse3". -# Default: host -cpuType: host - -# For a more general purpose virtual machine, Ubuntu container is optionally provided -# as a layer on the virtual machine. -# The underlying virtual machine is still accessible via `colima ssh --layer=false` or running `colima` in -# the Ubuntu session. -# -# Default: false -layer: false - -# Custom provision scripts for the virtual machine. -# Provisioning scripts are executed on startup and therefore needs to be idempotent. -# -# EXAMPLE - script exected as root -# provision: -# - mode: system -# script: apk add htop vim -# -# EXAMPLE - script exected as user -# provision: -# - mode: user -# script: | -# [ -f ~/.provision ] && exit 0; -# echo provisioning as $USER... -# touch ~/.provision -# -# Default: [] -provision: - - mode: system - script: | - mkdir -p /tmp/docker-bins - cd /tmp/docker-bins - wget -qO- "https://download.docker.com/linux/static/{{dockerChannel}}/{{hostArch}}/docker-{{dockerVersion}}.tgz" | tar xvz --strip 1 - mv -f /tmp/docker-bins/* /usr/bin/ - -# Modify ~/.ssh/config automatically to include a SSH config for the virtual machine. -# SSH config will still be generated in ~/.colima/ssh_config regardless. -# Default: true -sshConfig: false - -# Configure volume mounts for the virtual machine. -# Colima mounts user's home directory by default to provide a familiar -# user experience. -# -# EXAMPLE -# mounts: -# - location: ~/secrets -# writable: false -# - location: ~/projects -# writable: true -# -# Colima default behaviour: $HOME and /tmp/colima are mounted as writable. -# Default: [] -mounts: [] - -# Environment variables for the virtual machine. -# -# EXAMPLE -# env: -# KEY: value -# ANOTHER_KEY: another value -# -# Default: {} -env: {} diff --git a/scripts/setup-docker-linux.sh b/scripts/setup-docker-linux.sh deleted file mode 100755 index e09a765..0000000 --- a/scripts/setup-docker-linux.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2023 actions-toolkit authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu - -: "${TOOLDIR=}" -: "${RUNDIR=}" -: "${DOCKER_HOST=}" - -export PATH="$TOOLDIR::$PATH" - -if [ -z "$DOCKER_HOST" ]; then - echo >&2 'error: DOCKER_HOST required' - false -fi - -if ! command -v dockerd &> /dev/null; then - echo >&2 'error: dockerd missing from PATH' - false -fi - -mkdir -p "$RUNDIR" - -( - echo "Starting dockerd" - set -x - exec dockerd \ - --host="$DOCKER_HOST" \ - --exec-root="$RUNDIR/execroot" \ - --data-root="$RUNDIR/data" \ - --pidfile="$RUNDIR/docker.pid" \ - --userland-proxy=false \ - 2>&1 | tee "$RUNDIR/dockerd.log" -) & - -tries=60 -while ! docker version &> /dev/null; do - ((tries--)) - if [ $tries -le 0 ]; then - if [ -z "$DOCKER_HOST" ]; then - echo >&2 "error: daemon failed to start" - else - echo >&2 "error: daemon at $DOCKER_HOST fails to 'docker version':" - docker version >&2 || true - fi - false - fi - sleep 2 -done diff --git a/scripts/setup-docker-win.ps1 b/scripts/setup-docker-win.ps1 deleted file mode 100644 index e4c28b9..0000000 --- a/scripts/setup-docker-win.ps1 +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2023 actions-toolkit authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[CmdletBinding()] -param( - [Parameter(Mandatory = $true)] - [string]$ToolDir, - - [Parameter(Mandatory = $true)] - [string]$RunDir, - - [Parameter(Mandatory = $true)] - [string]$DockerHost) - -$pwver = (Get-ItemProperty -Path HKLM:\SOFTWARE\Microsoft\PowerShell\3\PowerShellEngine -Name 'PowerShellVersion').PowerShellVersion -Write-Host "PowerShell version: $pwver" - -# Create run directory -New-Item -ItemType Directory "$RunDir" -ErrorAction SilentlyContinue | Out-Null - -# Remove existing service -if (Get-Service docker -ErrorAction SilentlyContinue) { - $dockerVersion = (docker version -f "{{.Server.Version}}") - Write-Host "Current installed Docker version: $dockerVersion" - # stop service - Stop-Service -Force -Name docker - Write-Host "Service stopped" - # remove service - sc.exe delete "docker" - # removes event log entry. we could use "Remove-EventLog -LogName -Source docker" - # but this cmd is not available atm - $ErrorActionPreference = "SilentlyContinue" - & reg delete "HKLM\SYSTEM\CurrentControlSet\Services\EventLog\Application\docker" /f 2>&1 | Out-Null - $ErrorActionPreference = "Stop" - Write-Host "Service removed" -} - -$env:DOCKER_HOST = $DockerHost -Write-Host "DOCKER_HOST: $env:DOCKER_HOST" - -Write-Host "Creating service" -New-Item -ItemType Directory "$RunDir\moby-root" -ErrorAction SilentlyContinue | Out-Null -New-Item -ItemType Directory "$RunDir\moby-exec" -ErrorAction SilentlyContinue | Out-Null -Start-Process -Wait -NoNewWindow "$ToolDir\dockerd" ` - -ArgumentList ` - "--host=$DockerHost", ` - "--data-root=$RunDir\moby-root", ` - "--exec-root=$RunDir\moby-exec", ` - "--pidfile=$RunDir\docker.pid", ` - "--register-service" -Write-Host "Starting service" -Start-Service -Name docker -Write-Host "Service started successfully!" - -$tries=20 -Write-Host "Waiting for Docker daemon to start..." -While ($true) { - $ErrorActionPreference = "SilentlyContinue" - & "$ToolDir\docker" version | Out-Null - $ErrorActionPreference = "Stop" - If ($LastExitCode -eq 0) { - break - } - $tries-- - If ($tries -le 0) { - Throw "Failed to get a response from Docker daemon" - } - Write-Host -NoNewline "." - Start-Sleep -Seconds 1 -} -Write-Host "Docker daemon started successfully!" - -Get-WinEvent -ea SilentlyContinue ` - -FilterHashtable @{ProviderName= "docker"; LogName = "application"} | - Sort-Object @{Expression="TimeCreated";Descending=$false} | - ForEach-Object {"$($_.TimeCreated.ToUniversalTime().ToString("o")) [$($_.LevelDisplayName)] $($_.Message)"} diff --git a/src/docker/assets.ts b/src/docker/assets.ts new file mode 100644 index 0000000..9e4328f --- /dev/null +++ b/src/docker/assets.ts @@ -0,0 +1,348 @@ +/** + * Copyright 2023 actions-toolkit authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import {Context} from '../context'; + +export const setupDockerLinuxSh = (): string => { + return get('docker-setup-linux.sh', setupDockerLinuxShData, '0755'); +}; + +export const setupDockerWinPs1 = (): string => { + return get('docker-setup-win.ps1', setupDockerWinPs1Data); +}; + +export const colimaYaml = (): string => { + return get('colima.yaml', colimaYamlData); +}; + +const get = (filename: string, data: string, mode?: string): string => { + const assetPath = Context.tmpName({ + template: `docker-asset-XXXXXX-${filename}`, + tmpdir: Context.tmpDir() + }); + fs.writeFileSync(assetPath, data); + if (mode) { + fs.chmodSync(assetPath, mode); + } + return assetPath; +}; + +export const setupDockerLinuxShData = ` +#!/usr/bin/env bash + +set -eu + +: "\${TOOLDIR=}" +: "\${RUNDIR=}" +: "\${DOCKER_HOST=}" + +export PATH="$TOOLDIR::$PATH" + +if [ -z "$DOCKER_HOST" ]; then + echo >&2 'error: DOCKER_HOST required' + false +fi + +if ! command -v dockerd &> /dev/null; then + echo >&2 'error: dockerd missing from PATH' + false +fi + +mkdir -p "$RUNDIR" + +( + echo "Starting dockerd" + set -x + exec dockerd \\ + --host="$DOCKER_HOST" \\ + --exec-root="$RUNDIR/execroot" \\ + --data-root="$RUNDIR/data" \\ + --pidfile="$RUNDIR/docker.pid" \\ + --userland-proxy=false \\ + 2>&1 | tee "$RUNDIR/dockerd.log" +) & + +tries=60 +while ! docker version &> /dev/null; do + ((tries--)) + if [ $tries -le 0 ]; then + if [ -z "$DOCKER_HOST" ]; then + echo >&2 "error: daemon failed to start" + else + echo >&2 "error: daemon at $DOCKER_HOST fails to 'docker version':" + docker version >&2 || true + fi + false + fi + sleep 2 +done +echo "Docker daemon started successfully!" +`; + +export const setupDockerWinPs1Data = ` +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$ToolDir, + + [Parameter(Mandatory = $true)] + [string]$RunDir, + + [Parameter(Mandatory = $true)] + [string]$DockerHost) + +$pwver = (Get-ItemProperty -Path HKLM:\\SOFTWARE\\Microsoft\\PowerShell\\3\\PowerShellEngine -Name 'PowerShellVersion').PowerShellVersion +Write-Host "PowerShell version: $pwver" + +# Create run directory +New-Item -ItemType Directory "$RunDir" -ErrorAction SilentlyContinue | Out-Null + +# Remove existing service +if (Get-Service docker -ErrorAction SilentlyContinue) { + $dockerVersion = (docker version -f "{{.Server.Version}}") + Write-Host "Current installed Docker version: $dockerVersion" + # stop service + Stop-Service -Force -Name docker + Write-Host "Service stopped" + # remove service + sc.exe delete "docker" + # removes event log entry. we could use "Remove-EventLog -LogName -Source docker" + # but this cmd is not available atm + $ErrorActionPreference = "SilentlyContinue" + & reg delete "HKLM\\SYSTEM\\CurrentControlSet\\Services\\EventLog\\Application\\docker" /f 2>&1 | Out-Null + $ErrorActionPreference = "Stop" + Write-Host "Service removed" +} + +$env:DOCKER_HOST = $DockerHost +Write-Host "DOCKER_HOST: $env:DOCKER_HOST" + +Write-Host "Creating service" +New-Item -ItemType Directory "$RunDir\\moby-root" -ErrorAction SilentlyContinue | Out-Null +New-Item -ItemType Directory "$RunDir\\moby-exec" -ErrorAction SilentlyContinue | Out-Null +Start-Process -Wait -NoNewWindow "$ToolDir\\dockerd" \` + -ArgumentList \` + "--host=$DockerHost", \` + "--data-root=$RunDir\\moby-root", \` + "--exec-root=$RunDir\\moby-exec", \` + "--pidfile=$RunDir\\docker.pid", \` + "--register-service" +Write-Host "Starting service" +Start-Service -Name docker +Write-Host "Service started successfully!" + +$tries=20 +Write-Host "Waiting for Docker daemon to start..." +While ($true) { + $ErrorActionPreference = "SilentlyContinue" + & "$ToolDir\\docker" version | Out-Null + $ErrorActionPreference = "Stop" + If ($LastExitCode -eq 0) { + break + } + $tries-- + If ($tries -le 0) { + Throw "Failed to get a response from Docker daemon" + } + Write-Host -NoNewline "." + Start-Sleep -Seconds 1 +} +Write-Host "Docker daemon started successfully!" + +Get-WinEvent -ea SilentlyContinue \` + -FilterHashtable @{ProviderName= "docker"; LogName = "application"} | + Sort-Object @{Expression="TimeCreated";Descending=$false} | + ForEach-Object {"$($_.TimeCreated.ToUniversalTime().ToString("o")) [$($_.LevelDisplayName)] $($_.Message)"} +`; + +export const colimaYamlData = ` +# Number of CPUs to be allocated to the virtual machine. +# Default: 2 +cpu: 2 + +# Size of the disk in GiB to be allocated to the virtual machine. +# NOTE: changing this has no effect after the virtual machine has been created. +# Default: 60 +disk: 60 + +# Size of the memory in GiB to be allocated to the virtual machine. +# Default: 2 +memory: 2 + +# Architecture of the virtual machine (x86_64, aarch64, host). +# Default: host +arch: host + +# Container runtime to be used (docker, containerd). +# Default: docker +runtime: docker + +# Kubernetes configuration for the virtual machine. +kubernetes: + enabled: false + +# Auto-activate on the Host for client access. +# Setting to true does the following on startup +# - sets as active Docker context (for Docker runtime). +# - sets as active Kubernetes context (if Kubernetes is enabled). +# Default: true +autoActivate: false + +# Network configurations for the virtual machine. +network: + # Assign reachable IP address to the virtual machine. + # NOTE: this is currently macOS only and ignored on Linux. + # Default: false + address: false + + # Custom DNS resolvers for the virtual machine. + # + # EXAMPLE + # dns: [8.8.8.8, 1.1.1.1] + # + # Default: [] + dns: [] + + # DNS hostnames to resolve to custom targets using the internal resolver. + # This setting has no effect if a custom DNS resolver list is supplied above. + # It does not configure the /etc/hosts files of any machine or container. + # The value can be an IP address or another host. + # + # EXAMPLE + # dnsHosts: + # example.com: 1.2.3.4 + dnsHosts: + host.docker.internal: host.lima.internal + + # Network driver to use (slirp, gvproxy), (requires vmType \`qemu\`) + # - slirp is the default user mode networking provided by Qemu + # - gvproxy is an alternative to VPNKit based on gVisor https://github.com/containers/gvisor-tap-vsock + # Default: gvproxy + driver: gvproxy + +# Forward the host's SSH agent to the virtual machine. +# Default: false +forwardAgent: false + +# Docker daemon configuration that maps directly to daemon.json. +# https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file. +# NOTE: some settings may affect Colima's ability to start docker. e.g. \`hosts\`. +# +# EXAMPLE - disable buildkit +# docker: +# features: +# buildkit: false +# +# EXAMPLE - add insecure registries +# docker: +# insecure-registries: +# - myregistry.com:5000 +# - host.docker.internal:5000 +# +# Colima default behaviour: buildkit enabled +# Default: {} +docker: {} + +# Virtual Machine type (qemu, vz) +# NOTE: this is macOS 13 only. For Linux and macOS <13.0, qemu is always used. +# +# vz is macOS virtualization framework and requires macOS 13 +# +# Default: qemu +vmType: qemu + +# Volume mount driver for the virtual machine (virtiofs, 9p, sshfs). +# +# virtiofs is limited to macOS and vmType \`vz\`. It is the fastest of the options. +# +# 9p is the recommended and the most stable option for vmType \`qemu\`. +# +# sshfs is faster than 9p but the least reliable of the options (when there are lots +# of concurrent reads or writes). +# +# Default: virtiofs (for vz), sshfs (for qemu) +mountType: 9p + +# The CPU type for the virtual machine (requires vmType \`qemu\`). +# Options available for host emulation can be checked with: \`qemu-system-$(arch) -cpu help\`. +# Instructions are also supported by appending to the cpu type e.g. "qemu64,+ssse3". +# Default: host +cpuType: host + +# For a more general purpose virtual machine, Ubuntu container is optionally provided +# as a layer on the virtual machine. +# The underlying virtual machine is still accessible via \`colima ssh --layer=false\` or running \`colima\` in +# the Ubuntu session. +# +# Default: false +layer: false + +# Custom provision scripts for the virtual machine. +# Provisioning scripts are executed on startup and therefore needs to be idempotent. +# +# EXAMPLE - script exected as root +# provision: +# - mode: system +# script: apk add htop vim +# +# EXAMPLE - script exected as user +# provision: +# - mode: user +# script: | +# [ -f ~/.provision ] && exit 0; +# echo provisioning as $USER... +# touch ~/.provision +# +# Default: [] +provision: + - mode: system + script: | + mkdir -p /tmp/docker-bins + cd /tmp/docker-bins + wget -qO- "https://download.docker.com/linux/static/{{dockerChannel}}/{{hostArch}}/docker-{{dockerVersion}}.tgz" | tar xvz --strip 1 + mv -f /tmp/docker-bins/* /usr/bin/ + +# Modify ~/.ssh/config automatically to include a SSH config for the virtual machine. +# SSH config will still be generated in ~/.colima/ssh_config regardless. +# Default: true +sshConfig: false + +# Configure volume mounts for the virtual machine. +# Colima mounts user's home directory by default to provide a familiar +# user experience. +# +# EXAMPLE +# mounts: +# - location: ~/secrets +# writable: false +# - location: ~/projects +# writable: true +# +# Colima default behaviour: $HOME and /tmp/colima are mounted as writable. +# Default: [] +mounts: [] + +# Environment variables for the virtual machine. +# +# EXAMPLE +# env: +# KEY: value +# ANOTHER_KEY: another value +# +# Default: {} +env: {} +`; diff --git a/src/docker/install.ts b/src/docker/install.ts index 9b20435..fb37ba0 100644 --- a/src/docker/install.ts +++ b/src/docker/install.ts @@ -23,9 +23,9 @@ import * as core from '@actions/core'; import * as io from '@actions/io'; import * as tc from '@actions/tool-cache'; -import * as scripts from '../scripts'; import {Exec} from '../exec'; import {Util} from '../util'; +import {colimaYamlData, setupDockerLinuxSh, setupDockerWinPs1} from './assets'; export class Install { public async download(version: string, channel?: string): Promise { @@ -65,6 +65,12 @@ export class Install { } public async install(toolDir: string, runDir: string, version: string, channel?: string): Promise { + if (toolDir.length == 0) { + throw new Error('toolDir must be set'); + } + if (runDir.length == 0) { + throw new Error('runDir must be set'); + } channel = channel || 'stable'; switch (os.platform()) { case 'darwin': { @@ -97,12 +103,7 @@ export class Install { } await core.group('Creating colima config', async () => { - const colimaCfg = handlebars.compile( - fs.readFileSync(scripts.colimaConfig, { - encoding: 'utf8', - flag: 'r' - }) - )({ + const colimaCfg = handlebars.compile(colimaYamlData)({ hostArch: Install.platformArch(), dockerVersion: version, dockerChannel: channel @@ -134,7 +135,7 @@ export class Install { await core.group('Install Docker daemon', async () => { const bashPath: string = await io.which('bash', true); - await Exec.exec('sudo', ['-E', bashPath, scripts.setupDockerLinux], { + await Exec.exec('sudo', ['-E', bashPath, setupDockerLinuxSh()], { env: Object.assign({}, process.env, { TOOLDIR: toolDir, RUNDIR: runDir, @@ -154,7 +155,7 @@ export class Install { private async installWindows(toolDir: string, runDir: string): Promise { const dockerHost = 'npipe:////./pipe/setup_docker_action'; - const setupCmd = await Util.powershellCommand(scripts.setupDockerWin, { + const setupCmd = await Util.powershellCommand(setupDockerWinPs1(), { ToolDir: toolDir, RunDir: runDir, DockerHost: dockerHost diff --git a/src/scripts.ts b/src/scripts.ts deleted file mode 100644 index 600c1e3..0000000 --- a/src/scripts.ts +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2023 actions-toolkit authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import path from 'path'; - -const scriptsPath = path.join(__dirname, '..', 'scripts'); - -export const setupDockerLinux = path.join(scriptsPath, 'setup-docker-linux.sh'); -export const setupDockerWin = path.join(scriptsPath, 'setup-docker-win.ps1'); -export const colimaConfig = path.join(scriptsPath, 'colima.yaml'); From ac9d9d9a1b287485ae60eab62d24e2ba7e9248fc Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 2 Mar 2023 13:48:24 +0100 Subject: [PATCH 11/12] docker: detach dockerd for linux install Signed-off-by: CrazyMax --- package.json | 5 +++-- src/docker/assets.ts | 16 ---------------- src/docker/install.ts | 38 ++++++++++++++++++++++++++++++++++++-- yarn.lock | 17 +++++++++++++++++ 4 files changed, 56 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index 71f6ba7..42ebba1 100644 --- a/package.json +++ b/package.json @@ -11,9 +11,9 @@ "prettier": "prettier --check \"./**/*.ts\"", "prettier:fix": "prettier --write \"./**/*.ts\"", "test": "jest", - "test:e2e": "jest -c jest.config.e2e.ts --runInBand --detectOpenHandles --forceExit", + "test:e2e": "jest -c jest.config.e2e.ts --runInBand --detectOpenHandles", "test-coverage": "jest --coverage", - "test-coverage:e2e": "jest --coverage -c jest.config.e2e.ts --runInBand --detectOpenHandles --forceExit" + "test-coverage:e2e": "jest --coverage -c jest.config.e2e.ts --runInBand --detectOpenHandles" }, "repository": { "type": "git", @@ -50,6 +50,7 @@ "@actions/http-client": "^2.0.1", "@actions/io": "^1.1.2", "@actions/tool-cache": "^2.0.1", + "async-retry": "^1.3.3", "csv-parse": "^5.3.5", "handlebars": "^4.7.7", "jwt-decode": "^3.1.2", diff --git a/src/docker/assets.ts b/src/docker/assets.ts index 9e4328f..a8f82db 100644 --- a/src/docker/assets.ts +++ b/src/docker/assets.ts @@ -75,22 +75,6 @@ mkdir -p "$RUNDIR" --userland-proxy=false \\ 2>&1 | tee "$RUNDIR/dockerd.log" ) & - -tries=60 -while ! docker version &> /dev/null; do - ((tries--)) - if [ $tries -le 0 ]; then - if [ -z "$DOCKER_HOST" ]; then - echo >&2 "error: daemon failed to start" - else - echo >&2 "error: daemon at $DOCKER_HOST fails to 'docker version':" - docker version >&2 || true - fi - false - fi - sleep 2 -done -echo "Docker daemon started successfully!" `; export const setupDockerWinPs1Data = ` diff --git a/src/docker/install.ts b/src/docker/install.ts index fb37ba0..5829bc5 100644 --- a/src/docker/install.ts +++ b/src/docker/install.ts @@ -14,9 +14,11 @@ * limitations under the License. */ +import * as child_process from 'child_process'; import fs from 'fs'; import os from 'os'; import path from 'path'; +import retry from 'async-retry'; import * as handlebars from 'handlebars'; import * as util from 'util'; import * as core from '@actions/core'; @@ -133,9 +135,12 @@ export class Install { private async installLinux(toolDir: string, runDir: string): Promise { const dockerHost = `unix://${path.join(runDir, 'docker.sock')}`; - await core.group('Install Docker daemon', async () => { + await core.group('Start Docker daemon', async () => { const bashPath: string = await io.which('bash', true); - await Exec.exec('sudo', ['-E', bashPath, setupDockerLinuxSh()], { + const proc = await child_process.spawn(`sudo -E ${bashPath} ${setupDockerLinuxSh()}`, [], { + detached: true, + shell: true, + stdio: ['ignore', process.stdout, process.stderr], env: Object.assign({}, process.env, { TOOLDIR: toolDir, RUNDIR: runDir, @@ -144,6 +149,35 @@ export class Install { [key: string]: string; } }); + proc.unref(); + await retry( + async bail => { + await Exec.getExecOutput(`docker version`, undefined, { + ignoreReturnCode: true, + silent: true, + env: Object.assign({}, process.env, { + DOCKER_HOST: dockerHost + }) as { + [key: string]: string; + } + }) + .then(res => { + if (res.stderr.length > 0 && res.exitCode != 0) { + bail(new Error(res.stderr)); + return false; + } + return res.exitCode == 0; + }) + .catch(error => { + bail(error); + return false; + }); + }, + { + retries: 5 + } + ); + core.info(`Docker daemon started started successfully`); }); await core.group('Create Docker context', async () => { diff --git a/yarn.lock b/yarn.lock index 5a5562a..d9ea642 100644 --- a/yarn.lock +++ b/yarn.lock @@ -774,6 +774,7 @@ __metadata: "@types/tmp": ^0.2.3 "@typescript-eslint/eslint-plugin": ^5.49.0 "@typescript-eslint/parser": ^5.49.0 + async-retry: ^1.3.3 cpy-cli: ^4.2.0 csv-parse: ^5.3.5 dotenv: ^16.0.3 @@ -2034,6 +2035,15 @@ __metadata: languageName: node linkType: hard +"async-retry@npm:^1.3.3": + version: 1.3.3 + resolution: "async-retry@npm:1.3.3" + dependencies: + retry: 0.13.1 + checksum: 38a7152ff7265a9321ea214b9c69e8224ab1febbdec98efbbde6e562f17ff68405569b796b1c5271f354aef8783665d29953f051f68c1fc45306e61aec82fdc4 + languageName: node + linkType: hard + "asynckit@npm:^0.4.0": version: 0.4.0 resolution: "asynckit@npm:0.4.0" @@ -5947,6 +5957,13 @@ __metadata: languageName: node linkType: hard +"retry@npm:0.13.1": + version: 0.13.1 + resolution: "retry@npm:0.13.1" + checksum: 47c4d5be674f7c13eee4cfe927345023972197dbbdfba5d3af7e461d13b44de1bfd663bfc80d2f601f8ef3fc8164c16dd99655a221921954a65d044a2fc1233b + languageName: node + linkType: hard + "retry@npm:^0.12.0": version: 0.12.0 resolution: "retry@npm:0.12.0" From 70390c899b59d30df8c3ad000ad8182b9b460929 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 2 Mar 2023 15:25:29 +0100 Subject: [PATCH 12/12] docker: tearDown method Signed-off-by: CrazyMax --- __tests__/docker/install.test.e2e.ts | 1 + src/docker/assets.ts | 6 +++ src/docker/install.ts | 75 +++++++++++++++++++++++++--- src/util.ts | 8 +-- 4 files changed, 81 insertions(+), 9 deletions(-) diff --git a/__tests__/docker/install.test.e2e.ts b/__tests__/docker/install.test.e2e.ts index 243f0a8..054c8f2 100644 --- a/__tests__/docker/install.test.e2e.ts +++ b/__tests__/docker/install.test.e2e.ts @@ -33,6 +33,7 @@ describe('install', () => { await install.install(toolPath, tmpDir, version); await Docker.printVersion(); await Docker.printInfo(); + await install.tearDown(tmpDir); })()).resolves.not.toThrow(); }); }); diff --git a/src/docker/assets.ts b/src/docker/assets.ts index a8f82db..6ebed43 100644 --- a/src/docker/assets.ts +++ b/src/docker/assets.ts @@ -25,6 +25,10 @@ export const setupDockerWinPs1 = (): string => { return get('docker-setup-win.ps1', setupDockerWinPs1Data); }; +export const dockerServiceLogsPs1 = (): string => { + return get('docker-service-logs.ps1', dockerServiceLogsPs1Data); +}; + export const colimaYaml = (): string => { return get('colima.yaml', colimaYamlData); }; @@ -146,7 +150,9 @@ While ($true) { Start-Sleep -Seconds 1 } Write-Host "Docker daemon started successfully!" +`; +export const dockerServiceLogsPs1Data = ` Get-WinEvent -ea SilentlyContinue \` -FilterHashtable @{ProviderName= "docker"; LogName = "application"} | Sort-Object @{Expression="TimeCreated";Descending=$false} | diff --git a/src/docker/install.ts b/src/docker/install.ts index 5829bc5..8e73d7e 100644 --- a/src/docker/install.ts +++ b/src/docker/install.ts @@ -27,7 +27,7 @@ import * as tc from '@actions/tool-cache'; import {Exec} from '../exec'; import {Util} from '../util'; -import {colimaYamlData, setupDockerLinuxSh, setupDockerWinPs1} from './assets'; +import {colimaYamlData, dockerServiceLogsPs1, setupDockerLinuxSh, setupDockerWinPs1} from './assets'; export class Install { public async download(version: string, channel?: string): Promise { @@ -134,6 +134,7 @@ export class Install { private async installLinux(toolDir: string, runDir: string): Promise { const dockerHost = `unix://${path.join(runDir, 'docker.sock')}`; + await io.mkdirP(runDir); await core.group('Start Docker daemon', async () => { const bashPath: string = await io.which('bash', true); @@ -189,13 +190,15 @@ export class Install { private async installWindows(toolDir: string, runDir: string): Promise { const dockerHost = 'npipe:////./pipe/setup_docker_action'; - const setupCmd = await Util.powershellCommand(setupDockerWinPs1(), { - ToolDir: toolDir, - RunDir: runDir, - DockerHost: dockerHost - }); await core.group('Install Docker daemon service', async () => { + const setupCmd = await Util.powershellCommand(setupDockerWinPs1(), { + ToolDir: toolDir, + RunDir: runDir, + DockerHost: dockerHost + }); await Exec.exec(setupCmd.command, setupCmd.args); + const logCmd = await Util.powershellCommand(dockerServiceLogsPs1()); + await Exec.exec(logCmd.command, logCmd.args); }); await core.group('Create Docker context', async () => { @@ -204,6 +207,66 @@ export class Install { }); } + public async tearDown(runDir: string): Promise { + switch (os.platform()) { + case 'darwin': { + await this.tearDownDarwin(runDir); + break; + } + case 'linux': { + await this.tearDownLinux(runDir); + break; + } + case 'win32': { + await this.tearDownWindows(); + break; + } + default: { + throw new Error(`Unsupported platform: ${os.platform()}`); + } + } + } + + private async tearDownDarwin(runDir: string): Promise { + await core.group('Docker daemon logs', async () => { + await Exec.exec('colima', ['exec', '--', 'cat', '/var/log/docker.log']); + }); + await core.group('Stopping colima', async () => { + await Exec.exec('colima', ['stop', '--very-verbose']); + }); + await core.group('Removing Docker context', async () => { + await Exec.exec('docker', ['context', 'rm', '-f', 'setup-docker-action']); + }); + await core.group(`Cleaning up runDir`, async () => { + await Exec.exec('sudo', ['rm', '-rf', runDir]); + }); + } + + private async tearDownLinux(runDir: string): Promise { + await core.group('Docker daemon logs', async () => { + core.info(fs.readFileSync(path.join(runDir, 'dockerd.log'), {encoding: 'utf8'})); + }); + await core.group('Stopping Docker daemon', async () => { + await Exec.exec('sudo', ['kill', fs.readFileSync(path.join(runDir, 'docker.pid')).toString().trim()]); + }); + await core.group('Removing Docker context', async () => { + await Exec.exec('docker', ['context', 'rm', '-f', 'setup-docker-action']); + }); + await core.group(`Cleaning up runDir`, async () => { + await Exec.exec('sudo', ['rm', '-rf', runDir]); + }); + } + + private async tearDownWindows(): Promise { + await core.group('Docker daemon logs', async () => { + const logCmd = await Util.powershellCommand(dockerServiceLogsPs1()); + await Exec.exec(logCmd.command, logCmd.args); + }); + await core.group('Removing Docker context', async () => { + await Exec.exec('docker', ['context', 'rm', '-f', 'setup-docker-action']); + }); + } + private downloadURL(version: string, channel: string): string { const platformOS = Install.platformOS(); const platformArch = Install.platformArch(); diff --git a/src/util.ts b/src/util.ts index bed6ee5..86c57cb 100644 --- a/src/util.ts +++ b/src/util.ts @@ -74,12 +74,14 @@ export class Util { return true; } - public static async powershellCommand(script: string, params: Record) { + public static async powershellCommand(script: string, params?: Record) { const powershellPath: string = await io.which('powershell', true); const escapedScript = script.replace(/'/g, "''").replace(/"|\n|\r/g, ''); const escapedParams: string[] = []; - for (const key in params) { - escapedParams.push(`-${key} '${params[key].replace(/'/g, "''").replace(/"|\n|\r/g, '')}'`); + if (params) { + for (const key in params) { + escapedParams.push(`-${key} '${params[key].replace(/'/g, "''").replace(/"|\n|\r/g, '')}'`); + } } return { command: `"${powershellPath}"`,