From 5073234a813a7ab0ca3a6853bc074d64e795b97a Mon Sep 17 00:00:00 2001 From: Christian Dupuis Date: Mon, 17 Oct 2022 13:32:19 +0200 Subject: [PATCH] Add query support for index Signed-off-by: Christian Dupuis --- .gitignore | 1 + Taskfile.yaml | 27 ++++++ commands/detect.go | 12 ++- ...ed_skills.edn => enabled_skills_query.edn} | 0 query/index.go | 93 +++++++++++++++++++ query/query.go | 71 ++++++++++---- query/repository_query.edn | 35 +++++++ 7 files changed, 218 insertions(+), 21 deletions(-) create mode 100644 Taskfile.yaml rename query/{enabled_skills.edn => enabled_skills_query.edn} (100%) create mode 100644 query/index.go create mode 100644 query/repository_query.edn diff --git a/.gitignore b/.gitignore index 5ab9a2d..ef08581 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ dist/ /base-cli-plugin /go-skill.iml +/.idea diff --git a/Taskfile.yaml b/Taskfile.yaml new file mode 100644 index 0000000..0050a50 --- /dev/null +++ b/Taskfile.yaml @@ -0,0 +1,27 @@ +version: '3' + +tasks: + go:build: + cmds: + - go build -ldflags="-w -s -X 'github.com/docker/base-cli-plugin/internal.version={{.GIT_COMMIT}}'" + env: + CGO_ENABLED: 0 + vars: + GIT_COMMIT: + sh: git describe --tags | cut -c 2- + + go:install: + deps: [go:build] + cmds: + - mkdir -p ~/.docker/cli-plugins + - install base-cli-plugin ~/.docker/cli-plugins/docker-base + + go:fmt: + cmds: + - goimports -w . + - gofmt -w . + - go mod tidy + + go:release: + cmds: + - goreleaser release --rm-dist diff --git a/commands/detect.go b/commands/detect.go index 5c525ac..05e4652 100644 --- a/commands/detect.go +++ b/commands/detect.go @@ -60,10 +60,12 @@ func Detect(dockerCli command.Cli, image string, workspace string, apiKey string chainId := identity.ChainID(chainIds) s.Suffix = fmt.Sprintf(" Finding matching base images for %s", label) s.Restart() - images, err := query.ForBaseImage(chainId, workspace, apiKey) - - if err != nil { - return err + images, err := query.ForBaseImageInDb(chainId, workspace, apiKey) + if err != nil || images == nil { + images, err = query.ForBaseImageInIndex(chainId, workspace, apiKey) + if err != nil { + return err + } } if images != nil { bi := make([]string, len(*images)) @@ -199,5 +201,5 @@ func renderVulnerabilities(image query.Image) string { return strings.Join(parts, " ") + " " } } - return "" + return " no CVE data available " } diff --git a/query/enabled_skills.edn b/query/enabled_skills_query.edn similarity index 100% rename from query/enabled_skills.edn rename to query/enabled_skills_query.edn diff --git a/query/index.go b/query/index.go new file mode 100644 index 0000000..dbc00b7 --- /dev/null +++ b/query/index.go @@ -0,0 +1,93 @@ +/* + * Copyright © 2022 Docker, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package query + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +type IndexImage struct { + Digest string `json:"digest"` + CreatedAt time.Time `json:"createdAt"` + Platform struct { + Os string `json:"os"` + Arch string `json:"arch"` + Variant string `json:"variant"` + } `json:"platform"` + Layers []struct { + Digest string `json:"digest"` + Size int `json:"size"` + LastModified time.Time `json:"lastModified"` + } `json:"layers"` + DigestChainId string `json:"digestChainId"` + DiffIdChainId string `json:"diffIdChainId"` +} + +type IndexManifestList struct { + Name string `json:"name"` + Tags []string `json:"tags"` + Digest string `json:"digest"` + Images []IndexImage `json:"images"` +} + +func ForBaseImageInIndex(digest digest.Digest, workspace string, apiKey string) (*[]Image, error) { + url := fmt.Sprintf("https://api.dso.docker.com/docker-images/chain-ids/%s.json", digest.String()) + + resp, err := http.Get(url) + if err != nil { + return nil, errors.Wrapf(err, "failed to query index") + } + + if resp.StatusCode == 200 { + var manifestList []IndexManifestList + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrapf(err, "failed to read response body") + } + err = json.Unmarshal(body, &manifestList) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal response body") + } + var ii IndexImage + for _, i := range manifestList[0].Images { + if i.DigestChainId == digest.String() || i.DiffIdChainId == digest.String() { + ii = i + break + } + } + repository, err := ForRepositoryInDb(manifestList[0].Name, workspace, apiKey) + if err != nil { + return nil, errors.Wrapf(err, "failed to query for respository") + } + image := Image{ + Digest: ii.Digest, + CreatedAt: ii.CreatedAt, + Tags: manifestList[0].Tags, + Repository: *repository, + } + return &[]Image{image}, nil + } + + return nil, nil +} diff --git a/query/query.go b/query/query.go index 60fc13c..5e244be 100644 --- a/query/query.go +++ b/query/query.go @@ -43,6 +43,13 @@ type Report struct { Unspecified int64 `edn:"vulnerability.report/unspecified"` } +type Repository struct { + Badge string `edn:"docker.repository/badge"` + Host string `edn:"docker.repository/host"` + Name string `edn:"docker.repository/name"` + SupportedTags []string `edn:"docker.repository/supported-tags"` +} + type Image struct { TeamId string `edn:"atomist/team-id"` Digest string `edn:"docker.image/digest"` @@ -52,13 +59,8 @@ type Image struct { Name string `edn:"docker.tag/name"` } `edn:"docker.image/tag"` ManifestList []ManifestList `edn:"docker.image/manifest-list"` - Repository struct { - Badge string `edn:"docker.repository/badge"` - Host string `edn:"docker.repository/host"` - Name string `edn:"docker.repository/name"` - SupportedTags []string `edn:"docker.repository/supported-tags"` - } `edn:"docker.image/repository"` - File struct { + Repository Repository `edn:"docker.image/repository"` + File struct { Path string `edn:"git.file/path"` } `edn:"docker.image/file"` Commit struct { @@ -73,16 +75,25 @@ type Image struct { Report []Report `edn:"vulnerability.report/report"` } -type QueryResult struct { +type ImageQueryResult struct { Query struct { Data [][]Image `edn:"data"` } `edn:"query"` } +type RepositoryQueryResult struct { + Query struct { + Data [][]Repository `edn:"data"` + } `edn:"query"` +} + //go:embed base_image_query.edn var baseImageQuery string -//go:embed enabled_skills.edn +//go:embed repository_query.edn +var repositoryQuery string + +//go:embed enabled_skills_query.edn var enabledSkillsQuery string func CheckAuth(workspace string, apiKey string) bool { @@ -93,12 +104,12 @@ func CheckAuth(workspace string, apiKey string) bool { return true } -// ForBaseImage returns images with matching digest in :docker.image/blob-digest or :docker.image/diff-chain-id -func ForBaseImage(digest digest.Digest, workspace string, apiKey string) (*[]Image, error) { +// ForBaseImageInDb returns images with matching digest in :docker.image/blob-digest or :docker.image/diff-chain-id +func ForBaseImageInDb(digest digest.Digest, workspace string, apiKey string) (*[]Image, error) { resp, err := query(fmt.Sprintf(baseImageQuery, digest), workspace, apiKey) if workspace == "" || apiKey == "" { - var result QueryResult + var result ImageQueryResult err = edn.NewDecoder(resp.Body).Decode(&result) if err != nil { return nil, errors.Wrapf(err, "failed to unmarshal response") @@ -120,7 +131,7 @@ func ForBaseImage(digest digest.Digest, workspace string, apiKey string) (*[]Ima for _, img := range images { tba := true for j := range image { - if image[j].Digest == img[0].Digest && img[0].TeamId == "A0GLG1QQA" { + if image[j].Digest == img[0].Digest && img[0].TeamId == "A11PU8L1C" { image[j] = img[0] tba = false break @@ -138,10 +149,38 @@ func ForBaseImage(digest digest.Digest, workspace string, apiKey string) (*[]Ima } } -func query(query string, workspace string, apiKey string) (*http.Response, error) { - url := "https://api.atomist.com/datalog/team/" + workspace +func ForRepositoryInDb(repo string, workspace string, apiKey string) (*Repository, error) { + resp, err := query(fmt.Sprintf(repositoryQuery, repo), workspace, apiKey) + if workspace == "" || apiKey == "" { - url = "https://api.atomist.com/datalog/shared-vulnerability/queries" + var result RepositoryQueryResult + err = edn.NewDecoder(resp.Body).Decode(&result) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal response") + } + if len(result.Query.Data) > 0 { + return &result.Query.Data[0][0], nil + } else { + return nil, nil + } + } else { + var repositories [][]Repository + err = edn.NewDecoder(resp.Body).Decode(&repositories) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal response") + } + if len(repositories) > 0 { + return &repositories[0][0], nil + } else { + return nil, nil + } + } +} + +func query(query string, workspace string, apiKey string) (*http.Response, error) { + url := "https://api.dso.docker.com/datalog/team/" + workspace + if workspace == "" || apiKey == "" { + url = "https://api.dso.docker.com/datalog/shared-vulnerability/queries" query = fmt.Sprintf(`{:queries [{:name "query" :query %s}]}`, query) } else { query = fmt.Sprintf(`{:query %s}`, query) diff --git a/query/repository_query.edn b/query/repository_query.edn new file mode 100644 index 0000000..9af29a6 --- /dev/null +++ b/query/repository_query.edn @@ -0,0 +1,35 @@ +;; Copyright © 2022 Docker, Inc. +;; +;; Licensed under the Apache License, Version 2.0 (the "License"); +;; you may not use this file except in compliance with the License. +;; You may obtain a copy of the License at +;; +;; http://www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. + +[:find + ?repo + :in $ $before-db %% ?ctx + :where + [(ground "%s") ?name] + [(adb/query (quote [:find + (pull ?repo [:atomist/team-id + :docker.repository/host + :docker.repository/badge + :docker.repository/supported-tags + (:docker.repository/repository :as :docker.repository/name)]) + :in $ $b %% ?ctx [?name] + :where + [?repo :docker.repository/repository ?name] + [?repo :docker.repository/host "hub.docker.com"] + ]) + ?name) + ?results] + [(untuple ?results) [?result ...]] + [(untuple ?result) [?repo]] + ]