From 4ca962b70c81c5fd4943aa569b8559133869f94a Mon Sep 17 00:00:00 2001 From: Jonny Stoten Date: Fri, 27 Sep 2024 12:32:24 +0100 Subject: [PATCH] Add function for parsing DOI definition files (#172) Add a Rego builtin called `attest.internals.parse_library_definition` for parsing the DOI definition files in https://github.com/docker-library/official-images/tree/master/library. This will allow us to verify DOI provenance fields against these files which are the source of truth for DOI images. This function just defers to https://github.com/docker-library/bashbrew/blob/master/manifest/rfc2822.go. --- go.mod | 5 + go.sum | 15 +++ policy/rego.go | 107 ++++++++++++------ policy/rego_test.go | 26 ++++- .../test/def_parse/def_parse_test.rego | 18 +++ .../policies/test/{ => fetch}/fetch.rego | 0 .../policies/test/fetch/fetch_test.rego | 9 ++ policy/testdata/policies/test/fetch_test.rego | 7 -- 8 files changed, 144 insertions(+), 43 deletions(-) create mode 100644 policy/testdata/policies/test/def_parse/def_parse_test.rego rename policy/testdata/policies/test/{ => fetch}/fetch.rego (100%) create mode 100644 policy/testdata/policies/test/fetch/fetch_test.rego delete mode 100644 policy/testdata/policies/test/fetch_test.rego diff --git a/go.mod b/go.mod index b6c3e32..35349a7 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/containerd/containerd/v2 v2.0.0-rc.4 github.com/containerd/platforms v0.2.1 github.com/distribution/reference v0.6.0 + github.com/docker-library/bashbrew v0.1.12 github.com/go-openapi/runtime v0.28.0 github.com/go-openapi/strfmt v0.23.0 github.com/google/go-containerregistry v0.20.2 @@ -39,6 +40,7 @@ require ( cloud.google.com/go/iam v1.2.0 // indirect cloud.google.com/go/kms v1.19.0 // indirect cloud.google.com/go/longrunning v0.6.0 // indirect + github.com/Microsoft/hcsshim v0.12.6 // indirect github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect @@ -62,6 +64,7 @@ require ( github.com/blang/semver v3.5.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.3.8 // indirect + github.com/containerd/containerd v1.7.21 // indirect github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect @@ -177,4 +180,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect + pault.ag/go/debian v0.12.0 // indirect + pault.ag/go/topsort v0.1.1 // indirect ) diff --git a/go.sum b/go.sum index 0cffd97..1961ab2 100644 --- a/go.sum +++ b/go.sum @@ -54,10 +54,13 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hcsshim v0.12.6 h1:qEnZjoHXv+4/s0LmKZWE0/AiZmMWEIkFfWBSf1a0wlU= +github.com/Microsoft/hcsshim v0.12.6/go.mod h1:ZABCLVcvLMjIkzr9rUGcQ1QA0p0P3Ps+d3N1g2DsFfk= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= @@ -172,6 +175,8 @@ github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUo github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ= github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w= +github.com/containerd/containerd v1.7.21 h1:USGXRK1eOC/SX0L195YgxTHb0a00anxajOzgfN0qrCA= +github.com/containerd/containerd v1.7.21/go.mod h1:e3Jz1rYRUZ2Lt51YrH9Rz0zPyJBOlSvB3ghr2jbVD8g= github.com/containerd/containerd/v2 v2.0.0-rc.4 h1:Bvto4h5i2VZkQ+L5SrGupg5ilQ+zkVPILdjf9RWMego= github.com/containerd/containerd/v2 v2.0.0-rc.4/go.mod h1:p35nJi4Pl9ibzuoVOPc3MputVh6Gbp9xoDg9VHz6/YI= github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= @@ -208,6 +213,8 @@ github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker-library/bashbrew v0.1.12 h1:qykd2fxTMiudN/70XItEQqgk/7LeVoDiBTEnKTpkst8= +github.com/docker-library/bashbrew v0.1.12/go.mod h1:6fyRRSm4vgBAgTw87EsfOT7wXKsc4JA9I5cdQJmwOm8= github.com/docker/cli v27.1.1+incompatible h1:goaZxOqs4QKxznZjjBWKONQci/MywhtRv2oNn0GkeZE= github.com/docker/cli v27.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= @@ -401,6 +408,7 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d/go.mod h1:phT/jsRPBAEqjAibu1BurrabCBNTYiVI+zbmyCZJY6Q= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -591,6 +599,7 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMc github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/yashtewari/glob-intersection v0.2.0 h1:8iuHdN88yYuCzCdjt0gDe+6bAhUwBeEWqThExu54RFg= github.com/yashtewari/glob-intersection v0.2.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -632,6 +641,7 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= @@ -806,6 +816,11 @@ k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/A k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +pault.ag/go/debian v0.12.0 h1:b8ctSdBSGJ98NE1VLn06aSx70EUpczlP2qqSHEiYYJA= +pault.ag/go/debian v0.12.0/go.mod h1:UbnMr3z/KZepjq7VzbYgBEfz8j4+Pyrm2L5X1fzhy/k= +pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a/go.mod h1:INqx0ClF7kmPAMk2zVTX8DRnhZ/yaA/Mg52g8KFKE7k= +pault.ag/go/topsort v0.1.1 h1:L0QnhUly6LmTv0e3DEzbN2q6/FGgAcQvaEw65S53Bg4= +pault.ag/go/topsort v0.1.1/go.mod h1:r1kc/L0/FZ3HhjezBIPaNVhkqv8L0UJ9bxRuHRVZ0q4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/release-utils v0.8.4 h1:4QVr3UgbyY/d9p74LBhg0njSVQofUsAZqYOzVZBhdBw= diff --git a/policy/rego.go b/policy/rego.go index a879c32..ca4b8f9 100644 --- a/policy/rego.go +++ b/policy/rego.go @@ -1,12 +1,14 @@ package policy import ( + "bytes" "context" "encoding/json" "fmt" "os" "path/filepath" + "github.com/docker-library/bashbrew/manifest" "github.com/docker/attest/attestation" intoto "github.com/in-toto/in-toto-golang/in_toto" "github.com/open-policy-agent/opa/ast" @@ -149,6 +151,12 @@ var attestDecl = &ast.Builtin{ Nondeterministic: true, } +var internalParseLibraryDefinitionDecl = &ast.Builtin{ + Name: "attest.internals.parse_library_definition", + Decl: types.NewFunction(types.Args(types.S), dynamicObj), + Nondeterministic: false, +} + func wrapFunctionResult(value *ast.Term, err error) (*ast.Term, error) { var terms [][2]*ast.Term if err != nil { @@ -174,28 +182,50 @@ func handleErrors2(f func(rCtx rego.BuiltinContext, a, b *ast.Term) (*ast.Term, func RegoFunctions(regoOpts *RegoFnOpts) []*tester.Builtin { return []*tester.Builtin{ - { - Decl: verifyDecl, - Func: rego.Function2( - ®o.Function{ - Name: verifyDecl.Name, - Decl: verifyDecl.Decl, - Memoize: true, - Nondeterministic: verifyDecl.Nondeterministic, - }, - handleErrors2(regoOpts.verifyInTotoEnvelope)), - }, - { - Decl: attestDecl, - Func: rego.Function1( - ®o.Function{ - Name: attestDecl.Name, - Decl: attestDecl.Decl, - Memoize: true, - Nondeterministic: attestDecl.Nondeterministic, - }, - handleErrors1(regoOpts.fetchInTotoAttestations)), - }, + builtin2(verifyDecl, regoOpts.verifyInTotoEnvelope), + builtin1(attestDecl, regoOpts.fetchInTotoAttestations), + builtin1(internalParseLibraryDefinitionDecl, regoOpts.internalParseLibraryDefinition), + } +} + +func builtin1(decl *ast.Builtin, f rego.Builtin1) *tester.Builtin { + return &tester.Builtin{ + Decl: decl, + Func: rego.Function1( + ®o.Function{ + Name: decl.Name, + Decl: decl.Decl, + Memoize: true, + Nondeterministic: decl.Nondeterministic, + }, + handleErrors1(f)), + } +} + +func builtin2(decl *ast.Builtin, f rego.Builtin2) *tester.Builtin { + return &tester.Builtin{ + Decl: decl, + Func: rego.Function2( + ®o.Function{ + Name: decl.Name, + Decl: decl.Decl, + Memoize: true, + Nondeterministic: decl.Nondeterministic, + }, + handleErrors2(f)), + } +} + +type RegoFnOpts struct { + attestationResolver attestation.Resolver + attestationVerifier attestation.Verifier +} + +// this is exported for testing here and in clients of the library. +func NewRegoFunctionOptions(resolver attestation.Resolver, verifier attestation.Verifier) *RegoFnOpts { + return &RegoFnOpts{ + attestationResolver: resolver, + attestationVerifier: verifier, } } @@ -229,19 +259,6 @@ func (regoOpts *RegoFnOpts) fetchInTotoAttestations(rCtx rego.BuiltinContext, pr return set, nil } -type RegoFnOpts struct { - attestationResolver attestation.Resolver - attestationVerifier attestation.Verifier -} - -// this is exported for testing here and in clients of the library. -func NewRegoFunctionOptions(resolver attestation.Resolver, verifier attestation.Verifier) *RegoFnOpts { - return &RegoFnOpts{ - attestationResolver: resolver, - attestationVerifier: verifier, - } -} - // because we don't control the signature here (blame rego) // nolint:gocritic func (regoOpts *RegoFnOpts) verifyInTotoEnvelope(rCtx rego.BuiltinContext, envTerm, optsTerm *ast.Term) (*ast.Term, error) { @@ -285,6 +302,26 @@ func (regoOpts *RegoFnOpts) verifyInTotoEnvelope(rCtx rego.BuiltinContext, envTe return ast.NewTerm(value), nil } +// because we don't control the signature here (blame rego) +// nolint:gocritic +func (regoOpts *RegoFnOpts) internalParseLibraryDefinition(_ rego.BuiltinContext, definitionTerm *ast.Term) (*ast.Term, error) { + definitionStr, ok := definitionTerm.Value.(ast.String) + if !ok { + return nil, fmt.Errorf("predicateTypeTerm is not a string") + } + definition := string(definitionStr) + defBuffer := bytes.NewBufferString(definition) + parsed, err := manifest.Parse2822(defBuffer) + if err != nil { + return nil, err + } + value, err := ast.InterfaceToValue(parsed) + if err != nil { + return nil, err + } + return ast.NewTerm(value), nil +} + func loadYAML(path string, bs []byte) (interface{}, error) { var x interface{} bs, err := yaml.YAMLToJSON(bs) diff --git a/policy/rego_test.go b/policy/rego_test.go index 127154a..47cf304 100644 --- a/policy/rego_test.go +++ b/policy/rego_test.go @@ -12,7 +12,7 @@ import ( ) func TestPolicy(t *testing.T) { - paths := []string{"testdata/policies/test"} + paths := []string{"testdata/policies/test/fetch"} modules, store, err := tester.Load(paths, nil) require.NoError(t, err) resolver := &NullAttestationResolver{} @@ -35,6 +35,30 @@ func TestPolicy(t *testing.T) { assert.True(t, resolver.called) } +func TestPolicyDefParse(t *testing.T) { + paths := []string{"testdata/policies/test/def_parse"} + modules, store, err := tester.Load(paths, nil) + require.NoError(t, err) + resolver := &NullAttestationResolver{} + + opts := NewRegoFunctionOptions(resolver, nil) + ctx := context.Background() + ch, err := tester.NewRunner(). + SetStore(store). + AddCustomBuiltins(RegoFunctions(opts)). + CapturePrintOutput(true). + RaiseBuiltinErrors(true). + EnableTracing(true). + SetModules(modules). + RunTests(ctx, nil) + require.NoError(t, err) + require.NoError(t, err) + results := buffer(ch) + t.Log(string(results[0].Output)) + assert.Equalf(t, 1, len(results), "expected 1 results, got %d", len(results)) + assert.Truef(t, results[0].Pass(), "expected result 1 to pass, got %v", results[0].Location) +} + func buffer[T any](ch chan T) []T { var out []T for v := range ch { diff --git a/policy/testdata/policies/test/def_parse/def_parse_test.rego b/policy/testdata/policies/test/def_parse/def_parse_test.rego new file mode 100644 index 0000000..ea2cab4 --- /dev/null +++ b/policy/testdata/policies/test/def_parse/def_parse_test.rego @@ -0,0 +1,18 @@ +package def_parse_test + +import rego.v1 + +test_parse_library_definition if { + def := `Maintainers: me (@me) +GitRepo: blah + +Tags: 1, 2, 3 +GitCommit: fa105cb3c26c8f0e87d7dbb1bf5293691ac2f688 +File: Dockerfile.foo` + result := attest.internals.parse_library_definition(def) + definition := result.value + definition.Entries[0].GitRepo == "blah" + definition.Entries[0].GitCommit == "fa105cb3c26c8f0e87d7dbb1bf5293691ac2f688" + definition.Entries[0].Tags == ["1", "2", "3"] + definition.Entries[0].File == "Dockerfile.foo" +} diff --git a/policy/testdata/policies/test/fetch.rego b/policy/testdata/policies/test/fetch/fetch.rego similarity index 100% rename from policy/testdata/policies/test/fetch.rego rename to policy/testdata/policies/test/fetch/fetch.rego diff --git a/policy/testdata/policies/test/fetch/fetch_test.rego b/policy/testdata/policies/test/fetch/fetch_test.rego new file mode 100644 index 0000000..cfb61f5 --- /dev/null +++ b/policy/testdata/policies/test/fetch/fetch_test.rego @@ -0,0 +1,9 @@ +package attest_test + +import rego.v1 + +import data.attest + +test_sucess if { + attest.success +} diff --git a/policy/testdata/policies/test/fetch_test.rego b/policy/testdata/policies/test/fetch_test.rego deleted file mode 100644 index 4fbe1a4..0000000 --- a/policy/testdata/policies/test/fetch_test.rego +++ /dev/null @@ -1,7 +0,0 @@ -package attest - -import rego.v1 - -test_sucess if { - success -}