From f58c0ad05887d4c727c519e1480dbe96dfd447d9 Mon Sep 17 00:00:00 2001 From: James Carnegie Date: Thu, 9 Feb 2023 13:26:42 +0000 Subject: [PATCH] Reorg --- .github/workflows/build.yml | 8 +-- .github/workflows/release.yml | 9 +-- .gitignore | 9 +-- Dockerfile | 8 +-- README.md | 29 ++++---- bb.edn | 26 ------- deps.edn | 2 +- dev/user.clj | 71 +++++++++++++++---- docker/ops.go | 18 ++--- main.clj | 57 --------------- repository/atomisthq/tools.docker/0.1.0/go.sh | 4 -- .../atomisthq/tools.docker/0.1.0/manifest.edn | 24 ------- .../docker/babashka-pod-docker/0.1.0/go.sh | 3 + .../babashka-pod-docker/0.1.0/manifest.edn | 23 ++++++ 14 files changed, 120 insertions(+), 171 deletions(-) delete mode 100644 bb.edn delete mode 100644 main.clj delete mode 100755 repository/atomisthq/tools.docker/0.1.0/go.sh delete mode 100644 repository/atomisthq/tools.docker/0.1.0/manifest.edn create mode 100755 repository/docker/babashka-pod-docker/0.1.0/go.sh create mode 100644 repository/docker/babashka-pod-docker/0.1.0/manifest.edn diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 04e2b4e..3e381aa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,16 +1,12 @@ name: Build and push on: - workflow_dispatch: - pull_request: - branches: - - main push: branches: - main env: - IMAGE_NAME: docker/pod-atomisthq-docker + IMAGE_NAME: docker/babashka-pod-docker jobs: build: @@ -46,7 +42,7 @@ jobs: org.opencontainers.image.revision=${{ env.SHA }} tags: | latest - v0.${{ github.run_number }}.${{ github.run_attempt }} + 0.1.0 # Build and push Docker image with Buildx (don't push on PR) # https://github.com/docker/build-push-action diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cbda5e0..33204ab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,8 +1,9 @@ name: Release on: - release: - types: [created] + push: + branches: + - main jobs: releases-matrix: @@ -24,7 +25,7 @@ jobs: goos: ${{ matrix.goos }} goarch: ${{ matrix.goarch }} goversion: 1.19.1 - binary_name: "pod-atomisthq-tools.docker" - release_tag: ${{github.ref_name}} + binary_name: "babashka-pod-docker" + release_tag: "0.1.0" overwrite: TRUE compress_assets: OFF diff --git a/.gitignore b/.gitignore index 956ae06..c44f89a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,7 @@ +babashka-pod-docker +.calva/ /parser /.clj-kondo/ /.cpcache/ /.lsp/ -/pod-babashka-docker -/pod-babashka-docker-0.1.0-macos-aarch64.zip -/pod-atomisthq-docker -/pod-atomisthq-docker-0.1.0-macos-aarch64.zip /.nrepl-port -/pod-atomisthq-tools.docker -/pod-atomisthq-tools.docker-0.1.0-macos-aarch64.zip -/pod-atomisthq-tools.docker-0.1.0-macos-arm64.zip diff --git a/Dockerfile b/Dockerfile index a98590b..2cb3f6b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,10 +11,10 @@ COPY main.go ./ COPY docker/ ./docker/ COPY babashka/ ./babashka/ -RUN CGO_ENABLED=0 go build -o pod-atomisthq-tools.docker +RUN CGO_ENABLED=0 go build -o babashka-pod-docker FROM alpine:3.17 - +ARG version COPY repository/ /root/.babashka/pods/repository -COPY --from=build /app/pod-atomisthq-tools.docker /root/.babashka/pods/repository/atomisthq/tools.docker/0.1.0 -RUN chmod 755 /root/.babashka/pods/repository/atomisthq/tools.docker/0.1.0/pod-atomisthq-tools.docker +COPY --from=build /app/babashka-pod-docker /root/.babashka/pods/repository/docker/babashka-pod-docker/0.1.0 +RUN chmod 755 /root/.babashka/pods/repository/docker/babashka-pod-docker/0.1.0/babashka-pod-docker diff --git a/README.md b/README.md index a7462c2..e79a9eb 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,12 @@ This is a [babashka pod](https://github.com/babashka/pods) that binds some golan ```clojure (require '[babashka.pods :as pods]) -(pods/load-pod 'atomisthq/tools.docker "0.1.0") +(pods/load-pod 'docker/babashka-pod-docker "0.1.0") ; OR use a locally built pod binary -#_(pods/load-pod "./pod-atomisthq-tools.docker") +#_(pods/load-pod "./babashka-pod-docker") ;; load-pod will create this namespace with two vars -(require '[pod.atomisthq.docker :as docker]) +(require '[docker.babashka-pod-docker :as docker]) ;; parse image names using github.com/docker/distribution ;; turns golang structs into clojure maps @@ -31,25 +31,21 @@ This is a [babashka pod](https://github.com/babashka/pods) that binds some golan (docker/parse-dockerfile "FROM \\\n gcr.io/whatever:tag\nCMD [\"run\"]") ``` -Loading `'atomisthq/docker` from the pod registry will download the binary into `${user.home}/.babashka/pods/registry` (the `$BABASHKA_PODS_DIR` environment variable will be used if it exists). +Loading `'docker/babashka-pod-docker` from the pod registry will download the binary into `${user.home}/.babashka/pods/registry` (the `$BABASHKA_PODS_DIR` environment variable will be used if it exists). -## Building +## Building Locally To build the golang `parser` binary locally, run `go build`. ```bash -go build -o pod-babashka-docker -``` - -Create `vonwig/pod-atomisthq-tools.docker` which is a manifest list with pod binaries for both `amd64` and `arm64`. This image is a good way to pull the pod binaries into skill containers. - -```bash -bb build-pod-image +go build -o babashka-pod-docker ``` ## Releasing -Creating a release from a tag will trigger a build and release +All pushes to main will update the 0.1.0 release. This is becaus maintaining the pod version in the repository directory and in the pod registry is tricky. + +We hope to automate all of that in the future. ## Namespace generation @@ -83,11 +79,11 @@ Here is an example of bindings that will resolve at compile-time and go through ;; statically define dispatch functions - this is synchronous (defn parse [s] - (impl/invoke-public "pod.atomisthq.docker" "pod.atomisthq.docker/parse-dockerfile" [s] {})) + (impl/invoke-public "babashka-pod-docker" "babashka-pod-docker/parse-dockerfile" [s] {})) ;; async example (defn generate-sbom [s] - (impl/invoke-public "pod.atomisthq.docker" "pod.atomisthq.docker/-generate-sbom" + (impl/invoke-public "babashka-pod-docker" "babashka-pod-docker/-generate-sbom" [s cb] {:handlers {:done (fn []) :success cb @@ -95,8 +91,7 @@ Here is an example of bindings that will resolve at compile-time and go through ``` ``` -(pods/load-pod 'atomisthq/tools.docker "7.3.0") -(pods/load-pod "my-executable") +(pods/load-pod "/bin/babashka-pod-docker") ``` This method of dispatch does not require any dynamic namespace generation. diff --git a/bb.edn b/bb.edn deleted file mode 100644 index 3dd8598..0000000 --- a/bb.edn +++ /dev/null @@ -1,26 +0,0 @@ -{:tasks - {:requires ([babashka.fs :as fs]) - :init (do - (def n "pod-atomisthq-tools.docker") - (def os "macos") - (def version "0.1.0")) - check-for-builder {:task (-> (shell "docker buildx inspect buildx-multi-arch"))} - setup-builder {:task (if (= 1 (:exit check-for-builder)) - (shell {:continue true} "docker buildx create --name=buildx-multi-arch --driver=docker-container --driver-opt=network=host") - (println "buildx-multi-arch is already running")) - :depends [check-for-builder]} - build-pod-image {:task (shell "docker buildx build --builder=buildx-multi-arch --push --platform=linux/amd64,linux/arm64 --tag=vonwig/pod-atomisthq-tools.docker .") - :depends [setup-builder]} - - build (do (shell (format "go build -o %s" n)) - (fs/copy "pod-atomisthq-tools.docker" "/Users/slim/.babashka/pods/repository/atomisthq/tools.docker/0.1.0/" {:replace-existing true}) - (fs/copy "pod-atomisthq-tools.docker" "/Users/slim/.vscode/extensions/docker.slim-docker-lsp-client-0.0.1/" {:replace-existing true}) - (fs/copy "pod-atomisthq-tools.docker" "/Users/slim/kipz/docker-vscode-project-extension/" {:replace-existing true})) - aarch64 (do - (shell (format "zip %s-%s-%s-%s.zip %s" n version os "aarch64" n) {})) - x86 (do - (shell (format "zip %s-%s-%s-%s.zip %s" n version os "x86_64" n) {})) - linux-x86 - (do - (shell (format "zip %s-%s-%s-%s.zip %s" n version "linux" "amd64" n) {}))}} - diff --git a/deps.edn b/deps.edn index ce4aaff..ba77ed7 100644 --- a/deps.edn +++ b/deps.edn @@ -3,4 +3,4 @@ babashka/process {:mvn/version "0.4.13"} com.cognitect/transit-clj {:mvn/version "1.0.324"}} :aliases {:main {:extra-paths ["main"] - :exec-fn user1/transact-hashes}}} + :exec-fn user/transact-hashes}}} diff --git a/dev/user.clj b/dev/user.clj index ed310ba..49a42ca 100644 --- a/dev/user.clj +++ b/dev/user.clj @@ -1,26 +1,73 @@ (ns user (:require [babashka.pods :as pods] - [clojure.edn :as edn])) + [clojure.edn :as edn] + [babashka.pods.impl :as impl])) -(pods/load-pod 'atomisthq/tools.docker "0.1.0") -(require '[pod.atomisthq.docker :as docker]) +(defn load-pod + ([pod-spec] (load-pod pod-spec nil)) + ([pod-spec version opts] (load-pod pod-spec (assoc opts :version version))) + ([pod-spec opts] + (let [opts (if (string? opts) + {:version opts} + opts) + pod (impl/load-pod + pod-spec + (merge {:remove-ns remove-ns + :resolve (fn [sym] + (or (resolve sym) + (intern + (create-ns (symbol (namespace sym))) + (symbol (name sym)))))} + opts))] + (future (impl/processor pod)) + {:pod/id (:pod-id pod)}))) -;; parse image names using github.com/docker/distribution +(comment + (pods/load-pod 'docker/babashka-pod-docker "0.1.0") + + (require '[babashka-pod-docker :as docker]) + + +;; parse image names using github.com/docker/distribution ;; turns golang structs into clojure maps -(docker/parse-image-name "gcr.io/whatever:tag") + (docker/parse-image-name "gcr.io/whatever:tag") + ;; automatically turns golang errors into Exceptions -(try - (docker/parse-image-name "gcr.io/whatever/:tag") - (catch Exception e + (try + (docker/parse-image-name "gcr.io/whatever/:tag") + (catch Exception e ;; invalid reference format - (println (.getMessage e)))) + (println (.getMessage e)))) + ;; parse dockerfiles using github.com/moby/buildkit ;; returns the Result struct transformed to a clojure map -(docker/parse-dockerfile "FROM \\\n gcr.io/whatever:tag\nCMD [\"run\"]") + (docker/parse-dockerfile "FROM \\\n gcr.io/whatever:tag\nCMD [\"run\"]") + ;; run sbom generation on local image -(docker/sbom "vonwig/clojure-base:jdk17" (fn [event] (println event))) + (docker/sbom "vonwig/clojure-base:jdk17" (fn [event] (println event))) -(docker/hashes "vonwig/malware1:latest" (fn [event] (println event))) + (docker/hashes "vonwig/malware1:latest" (fn [event] (println event))) + ) + +(defn generate-sbom + [image] + (impl/invoke-public + "docker.babashka-pod-docker" + "babashka-pod-docker/generate-sbom" + [image "" ""] + {:handlers {:done (fn [] (println "Done")) + :success (fn [msg] (println "msg: " msg)) + :error (fn [_err] #_"TODO: handle this error")}})) + +(comment + (println (load-pod "./babashka-pod-docker")) + (impl/invoke-public + "docker.babashka-pod-docker" + "babashka-pod-docker/parse-image-name" + ["ubuntu:latest"] + {}) + (generate-sbom "alpine") + ) diff --git a/docker/ops.go b/docker/ops.go index b7b08ac..5363839 100644 --- a/docker/ops.go +++ b/docker/ops.go @@ -110,7 +110,7 @@ func ProcessMessage(message *babashka.Message) (any, error) { Format: "json", Namespaces: []babashka.Namespace{ { - Name: "pod.atomisthq.docker", + Name: "docker.babashka-pod-docker", Vars: []babashka.Var{ { Name: "parse-image-name", @@ -126,8 +126,8 @@ func ProcessMessage(message *babashka.Message) (any, error) { (sbom image cb {})) ([image cb opts] (babashka.pods/invoke - "pod.atomisthq.docker" - 'pod.atomisthq.docker/-generate-sbom + "docker.babashka-pod-docker" + 'babashka-pod-docker/generate-sbom [image] {:handlers {:success (fn [event] (cb event)) @@ -144,8 +144,8 @@ func ProcessMessage(message *babashka.Message) (any, error) { (hashes image cb {})) ([image cb opts] (babashka.pods/invoke - "pod.atomisthq.docker" - 'pod.atomisthq.docker/-generate-hashes + "docker.babashka-pod-docker" + 'babashka-pod-docker/generate-hashes [image] {:handlers {:success (fn [event] (cb event)) @@ -160,21 +160,21 @@ func ProcessMessage(message *babashka.Message) (any, error) { }, nil case "invoke": switch message.Var { - case "pod.atomisthq.docker/parse-image-name": + case "babashka-pod-docker/parse-image-name": args := []string{} if err := json.Unmarshal([]byte(message.Args), &args); err != nil { return nil, err } return parse_uri(args[0]) - case "pod.atomisthq.docker/parse-dockerfile": + case "babashka-pod-docker/parse-dockerfile": args := []string{} if err := json.Unmarshal([]byte(message.Args), &args); err != nil { return nil, err } reader := strings.NewReader(args[0]) return parser.Parse(reader) - case "pod.atomisthq.docker/-generate-sbom": + case "babashka-pod-docker/generate-sbom": args := []string{} if err := json.Unmarshal([]byte(message.Args), &args); err != nil { @@ -193,7 +193,7 @@ func ProcessMessage(message *babashka.Message) (any, error) { } return "done", nil - case "pod.atomisthq.docker/-generate-hashes": + case "babashka-pod-docker/generate-hashes": args := []string{} if err := json.Unmarshal([]byte(message.Args), &args); err != nil { return nil, err diff --git a/main.clj b/main.clj deleted file mode 100644 index 7480699..0000000 --- a/main.clj +++ /dev/null @@ -1,57 +0,0 @@ -(ns main - (:require [babashka.pods :as pods] - [clojure.edn :as edn] - [babashka.curl :as curl] - [clojure.string :as string])) - -(def x (pods/load-pod 'atomisthq/tools.docker "0.1.0")) -(require '[pod.atomisthq.docker :as docker]) - -(defn do-transaction [all-hashes transactions m token digest] - (let [tx-data (->> @all-hashes - (filter (fn [{:keys [path]}] (if path (string/includes? path ".exe")))) - (mapcat (fn [{:keys [hash diff-id]}] - (let [blob-digest (get m diff-id)] - (if blob-digest - [{:schema/entity blob-digest - :schema/entity-type :docker.image/blob - :docker.image.blob/digest blob-digest} - {:schema/entity-type :docker.image.blob/file - :docker.image.blob.file/sha256 hash - :docker.image.blob.file/blob blob-digest}] - (do - (println diff-id "not in " m) - []))))) - (into []))] - (try - (println "tx-data" tx-data) - (println - (curl/post transactions - {:body (pr-str {:transactions [{:data tx-data}]}) - :headers {"Authorization" (format "Bearer %s" token) - "Content-Type" "application/edn"}})) - (println - (curl/post transactions - {:body (pr-str {:transactions [{:data [{:docker.image/digest digest - :schema/entity-type :docker/image - :malware.status/indexed :malware.status.indexed/complete}]}]}) - :headers {"Authorization" (format "Bearer %s" token) - "Content-Type" "application/edn"}})) - (System/exit 0) - (catch Throwable t - (println "error " t) - (System/exit 1))))) - -(defn transact-hashes [{:keys [image digest m transactions token]}] - (println image digest transactions) - (let [all-hashes (atom [])] - (docker/hashes image (fn [event] - (if (= "done" (:status event)) - (do-transaction all-hashes transactions m token digest) - (swap! all-hashes conj (edn/read-string event))))))) - -#_(let [[image digest m transaction-url token] *command-line-args*] - (transact-hashes {:image image :digest digest :diff-id->digest (edn/read-string m) :transaction-url transaction-url :token token})) - -(transact-hashes (edn/read-string (slurp "/Users/slim/atmhq/malware/test1.edn"))) -(while true (Thread/sleep 5000)) diff --git a/repository/atomisthq/tools.docker/0.1.0/go.sh b/repository/atomisthq/tools.docker/0.1.0/go.sh deleted file mode 100755 index 0a96fba..0000000 --- a/repository/atomisthq/tools.docker/0.1.0/go.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -export ATOMIST_LOG_LEVEL=warn; $(dirname "$0")/pod-atomisthq-tools.docker - diff --git a/repository/atomisthq/tools.docker/0.1.0/manifest.edn b/repository/atomisthq/tools.docker/0.1.0/manifest.edn deleted file mode 100644 index c53a078..0000000 --- a/repository/atomisthq/tools.docker/0.1.0/manifest.edn +++ /dev/null @@ -1,24 +0,0 @@ -{:pod/name atomisthq/tools.docker - :pod/description "docker golang utils for clojure" - :pod/version "0.1.0" - :pod/license "Apache" - :pod/example "https://raw.githubusercontent.com/atomisthq/pod-atomisthq-tools.docker/main/dev/user.clj" - :pod/language "go" - :pod/artifacts - [{:os/name "Linux.*" - :os/arch "amd64" - :artifact/url "https://github.com/atomisthq/pod-atomisthq-tools.docker/releases/download/v0.1.0/pod-atomisthq-tools.docker-0.1.0-linux-amd64.zip" - :artifact/executable "go.sh"} - {:os/name "Mac.*" - :os/arch "x86_64" - :artifact/url "https://github.com/atomisthq/pod-atomisthq-tools.docker/releases/download/v0.1.0/pod-atomisthq-tools.docker-0.1.0-macos-x86_64.zip" - :artifact/executable "go.sh"} - {:os/name "Mac.*" - :os/arch "aarch64" - :artifact/url "https://github.com/atomisthq/pod-atomisthq-tools.docker/releases/download/v0.1.0/pod-atomisthq-tools.docker-0.1.0-macos-arm64.zip" - :artifact/executable "go.sh"} - {:os/name "Linux.*" - :os/arch "aarch64" - :artifact/url "https://github.com/atomisthq/pod-atomisthq-tools.docker/releases/download/v0.1.0/pod-atomisthq-tools.docker-0.1.0-linux-arm64.zip" - :artifact/executable "go.sh"}]} - diff --git a/repository/docker/babashka-pod-docker/0.1.0/go.sh b/repository/docker/babashka-pod-docker/0.1.0/go.sh new file mode 100755 index 0000000..53477bd --- /dev/null +++ b/repository/docker/babashka-pod-docker/0.1.0/go.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +export ATOMIST_LOG_LEVEL=warn; $(dirname "$0")/babashka-pod-docker diff --git a/repository/docker/babashka-pod-docker/0.1.0/manifest.edn b/repository/docker/babashka-pod-docker/0.1.0/manifest.edn new file mode 100644 index 0000000..fc00e60 --- /dev/null +++ b/repository/docker/babashka-pod-docker/0.1.0/manifest.edn @@ -0,0 +1,23 @@ +{:pod/name docker/babashka-pod-docker + :pod/description "docker golang utils for clojure" + :pod/version "0.1.0" + :pod/license "Apache" + :pod/example "https://raw.githubusercontent.com/docker/babashka-pod-docker/main/dev/user.clj" + :pod/language "go" + :pod/artifacts + [{:os/name "Linux.*" + :os/arch "amd64" + :artifact/url "https://github.com/docker/babashka-pod-docker/releases/download/v0.1.0/babashka-pod-docker-0.1.0-linux-amd64.zip" + :artifact/executable "go.sh"} + {:os/name "Mac.*" + :os/arch "x86_64" + :artifact/url "https://github.com/docker/babashka-pod-docker/releases/download/v0.1.0/babashka-pod-docker-0.1.0-macos-x86_64.zip" + :artifact/executable "go.sh"} + {:os/name "Mac.*" + :os/arch "aarch64" + :artifact/url "https://github.com/docker/babashka-pod-docker/releases/download/v0.1.0/babashka-pod-docker-0.1.0-macos-arm64.zip" + :artifact/executable "go.sh"} + {:os/name "Linux.*" + :os/arch "aarch64" + :artifact/url "https://github.com/docker/babashka-pod-docker/releases/download/v0.1.0/babashka-pod-docker-0.1.0-linux-arm64.zip" + :artifact/executable "go.sh"}]}