docker/attest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: combine tuf code

Browse Source
This commit is contained in:
mrjoelkamp
2024-04-15 15:20:56 -05:00
parent 95295bc736
commit c1035c951e
56 changed files with 2660 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

196
pkg/mirror/metadata.go Normal file
View File

@@ -0,0 +1,196 @@
package mirror
import (
"fmt"
"strconv"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/static"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/theupdateframework/go-tuf/v2/metadata"
)
// -----------------
// TUF root metadata
// -----------------
// GetMetadataManifest returns an image with TUF root metadata as layers
func (m *TufMirror) GetMetadataManifest(metadataURL string) (*v1.Image, error) {
metadata, err := m.getTufMetadataMirror(metadataURL)
if err != nil {
return nil, fmt.Errorf("failed to get metadata: %w", err)
}
manifest, err := m.buildMetadataManifest(metadata)
if err != nil {
return nil, fmt.Errorf("failed to build metadata manifest: %w", err)
}
return manifest, nil
}
// getTufMetadataMirror returns a TufMetadata struct with TUF metadata as map of file names to bytes
func (m *TufMirror) getTufMetadataMirror(metadataURL string) (*TufMetadata, error) {
trustedMetadata := m.TufClient.GetMetadata()
rootMetadata := map[string][]byte{}
rootVersion := trustedMetadata.Root.Signed.Version
// get the previous versions of root metadata if any
if rootVersion != 1 {
var err error
rootMetadata, err = m.TufClient.GetPriorRoots(metadataURL)
if err != nil {
return nil, fmt.Errorf("failed to get prior root metadata: %w", err)
}
}
// get current root metadata
rootBytes, err := trustedMetadata.Root.ToBytes(false)
if err != nil {
return nil, fmt.Errorf("failed to get root metadata: %w", err)
}
rootMetadata[nameFromRole(metadata.ROOT, strconv.FormatInt(rootVersion, 10))] = rootBytes
snapshotBytes, err := trustedMetadata.Snapshot.ToBytes(false)
if err != nil {
return nil, fmt.Errorf("failed to get snapshot metadata: %w", err)
}
targetsBytes, err := trustedMetadata.Targets[metadata.TARGETS].ToBytes(false)
if err != nil {
return nil, fmt.Errorf("failed to get targets metadata: %w", err)
}
timestampBytes, err := trustedMetadata.Timestamp.ToBytes(false)
if err != nil {
return nil, fmt.Errorf("failed to get timestamp metadata: %w", err)
}
snapshotVersion := ""
targetsVersion := ""
if trustedMetadata.Root.Signed.ConsistentSnapshot {
snapshotVersion = strconv.FormatInt(trustedMetadata.Snapshot.Signed.Version, 10)
targetsVersion = strconv.FormatInt(trustedMetadata.Targets[metadata.TARGETS].Signed.Version, 10)
}
return &TufMetadata{
Root: rootMetadata,
Snapshot: map[string][]byte{nameFromRole(metadata.SNAPSHOT, snapshotVersion): snapshotBytes},
Targets: map[string][]byte{nameFromRole(metadata.TARGETS, targetsVersion): targetsBytes},
Timestamp: timestampBytes,
}, nil
}
// buildMetadataManifest returns an OCI image with TUF metadata as layers with annotations
func (m *TufMirror) buildMetadataManifest(metadata *TufMetadata) (*v1.Image, error) {
img := empty.Image
img = mutate.MediaType(img, types.OCIManifestSchema1)
img = mutate.ConfigMediaType(img, types.OCIConfigJSON)
for _, role := range TufRoles {
layers, err := m.makeRoleLayers(role, metadata)
if err != nil {
return nil, fmt.Errorf("failed to make role layer: %w", err)
}
img, err = mutate.Append(img, *layers...)
if err != nil {
return nil, fmt.Errorf("failed to append role layer to image: %w", err)
}
}
return &img, nil
}
// makeRoleLayers returns a list of layers for a given TUF role
func (m *TufMirror) makeRoleLayers(role TufRole, tufMetadata *TufMetadata) (*[]mutate.Addendum, error) {
layers := new([]mutate.Addendum)
ann := map[string]string{tufFileAnnotation: ""}
switch role {
case metadata.ROOT:
layers = m.annotatedMetaLayers(tufMetadata.Root)
case metadata.SNAPSHOT:
layers = m.annotatedMetaLayers(tufMetadata.Snapshot)
case metadata.TARGETS:
layers = m.annotatedMetaLayers(tufMetadata.Targets)
case metadata.TIMESTAMP:
ann[tufFileAnnotation] = fmt.Sprintf("%s.json", role)
*layers = append(*layers, mutate.Addendum{Layer: static.NewLayer(tufMetadata.Timestamp, tufMetadataMediaType), Annotations: ann})
default:
return nil, fmt.Errorf("unsupported TUF role: %s", role)
}
return layers, nil
}
// annotatedMetaLayers returns a list of layers with annotations for each TUF metadata file
func (m *TufMirror) annotatedMetaLayers(meta map[string][]byte) *[]mutate.Addendum {
layers := new([]mutate.Addendum)
for name, data := range meta {
ann := map[string]string{tufFileAnnotation: name}
*layers = append(*layers, mutate.Addendum{Layer: static.NewLayer(data, tufMetadataMediaType), Annotations: ann})
}
return layers
}
// ------------------------------
// TUF delegated targets metadata
// ------------------------------
// GetDelegatedMetadataMirrors returns a list of mirrors (image/tag pairs) for each delegated targets role metadata
func (m *TufMirror) GetDelegatedMetadataMirrors() ([]*MirrorImage, error) {
// get current delegated targets metadata
delegatedTargets, err := m.getDelegatedTargetsMetadata()
if err != nil {
return nil, fmt.Errorf("failed to get delegated targets metadata: %w", err)
}
mirror, err := m.buildDelegatedMetadataManifests(delegatedTargets)
if err != nil {
return nil, fmt.Errorf("failed to build delegated targets manifests: %w", err)
}
return mirror, nil
}
// getDelegatedTargetsMetadata returns delegated targets metadata as a list of DelegatedTargetMetadata (role name and data)
func (m *TufMirror) getDelegatedTargetsMetadata() (*[]DelegatedTargetMetadata, error) {
delegatedTargets := new([]DelegatedTargetMetadata)
md := m.TufClient.GetMetadata()
for _, role := range md.Targets[metadata.TARGETS].Signed.Delegations.Roles {
roleMetadata, err := m.TufClient.LoadDelegatedTargets(role.Name, metadata.TARGETS)
if err != nil {
return nil, fmt.Errorf("failed to get delegated role metadata: %w", err)
}
roleBytes, err := roleMetadata.ToBytes(false)
if err != nil {
return nil, fmt.Errorf("failed to get role %s metadata: %w", role.Name, err)
}
meta, ok := md.Snapshot.Signed.Meta[nameFromRole(role.Name, "")]
if !ok {
return nil, fmt.Errorf("failed to get role %s metadata: %w", role.Name, err)
}
// extract target metadata version in case of consistent snapshot naming
version := ""
if md.Root.Signed.ConsistentSnapshot {
version = strconv.FormatInt(meta.Version, 10)
}
*delegatedTargets = append(*delegatedTargets, DelegatedTargetMetadata{Name: role.Name, Version: version, Data: roleBytes})
}
return delegatedTargets, nil
}
// buildDelegatedMetadataManifests returns a list of mirrors (image/tag pairs) for each delegated target role metadata
func (m *TufMirror) buildDelegatedMetadataManifests(delegated *[]DelegatedTargetMetadata) ([]*MirrorImage, error) {
manifests := []*MirrorImage{}
for _, role := range *delegated {
img := empty.Image
img = mutate.MediaType(img, types.OCIManifestSchema1)
img = mutate.ConfigMediaType(img, types.OCIConfigJSON)
ann := map[string]string{tufFileAnnotation: nameFromRole(role.Name, role.Version)}
layer := mutate.Addendum{Layer: static.NewLayer(role.Data, tufMetadataMediaType), Annotations: ann}
img, err := mutate.Append(img, layer)
if err != nil {
return nil, fmt.Errorf("failed to append delegated targets layer to image: %w", err)
}
manifests = append(manifests, &MirrorImage{Image: &img, Tag: role.Name})
}
return manifests, nil
}
func nameFromRole(role, version string) string {
if version != "" {
return fmt.Sprintf("%s.%s.json", version, role)
}
return fmt.Sprintf("%s.json", role)
}

89
pkg/mirror/metadata_test.go Normal file
View File

@@ -0,0 +1,89 @@
package mirror
import (
"encoding/json"
"net/http"
"net/http/httptest"
"path/filepath"
"strconv"
"strings"
"testing"
"github.com/docker/attest/internal/embed"
"github.com/docker/attest/internal/test"
"github.com/stretchr/testify/assert"
"github.com/theupdateframework/go-tuf/v2/metadata"
)
func TestGetTufMetadataMirror(t *testing.T) {
server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join("..", "..", "internal", "test", "testdata", "test-repo"))))
defer server.Close()
path := test.CreateTempDir(t, "", "tuf_temp")
m, err := NewTufMirror(embed.DevRoot, path, server.URL+"/metadata", server.URL+"/targets")
assert.Nil(t, err)
tufMetadata, err := m.getTufMetadataMirror(server.URL + "/metadata")
assert.Nil(t, err)
// check that all roles are not empty
assert.Greater(t, len(tufMetadata.Root), 0)
assert.Greater(t, len(tufMetadata.Snapshot), 0)
assert.Greater(t, len(tufMetadata.Targets), 0)
assert.Greater(t, len(tufMetadata.Timestamp), 0)
}
func TestGetMetadataManifest(t *testing.T) {
server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join("..", "..", "internal", "test", "testdata", "test-repo"))))
defer server.Close()
path := test.CreateTempDir(t, "", "tuf_temp")
m, err := NewTufMirror(embed.DevRoot, path, server.URL+"/metadata", server.URL+"/targets")
assert.Nil(t, err)
img, err := m.GetMetadataManifest(server.URL + "/metadata")
assert.Nil(t, err)
assert.NotNil(t, img)
image := *img
mf, err := image.RawManifest()
assert.Nil(t, err)
type Annotations struct {
Annotations map[string]string `json:"annotations"`
}
type Layers struct {
Layers []Annotations `json:"layers"`
}
l := &Layers{}
err = json.Unmarshal(mf, l)
assert.Nil(t, err)
// check that layers are annotated and use consistent snapshot naming
for _, layer := range l.Layers {
ann, ok := layer.Annotations[tufFileAnnotation]
assert.True(t, ok)
// check for consistent snapshot version
parts := strings.Split(ann, ".")
if parts[0] == metadata.TIMESTAMP {
continue
}
_, err := strconv.Atoi(parts[0])
assert.Nil(t, err)
}
}
func TestGetDelegatedMetadataMirrors(t *testing.T) {
server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join("..", "..", "internal", "test", "testdata", "test-repo"))))
defer server.Close()
path := test.CreateTempDir(t, "", "tuf_temp")
m, err := NewTufMirror(embed.DevRoot, path, server.URL+"/metadata", server.URL+"/targets")
assert.Nil(t, err)
delegations, err := m.GetDelegatedMetadataMirrors()
assert.Nil(t, err)
assert.NotNil(t, delegations)
assert.Greater(t, len(delegations), 0)
}

82
pkg/mirror/mirror.go Normal file
View File

@@ -0,0 +1,82 @@
package mirror
import (
"fmt"
"log"
"os"
"github.com/docker/attest/internal/embed"
"github.com/docker/attest/pkg/tuf"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/layout"
"github.com/google/go-containerregistry/pkg/v1/remote"
)
func NewTufMirror(root []byte, tufPath, metadataURL, targetsURL string) (*TufMirror, error) {
if root == nil {
root = embed.DefaultRoot
}
tufClient, err := tuf.NewTufClient(root, tufPath, metadataURL, targetsURL)
if err != nil {
return nil, fmt.Errorf("failed to create TUF client: %w", err)
}
return &TufMirror{TufClient: tufClient, tufPath: tufPath, metadataURL: metadataURL, targetsURL: targetsURL}, nil
}
func PushToRegistry(image any, imageName string) error {
// Parse the image name
ref, err := name.ParseReference(imageName)
if err != nil {
log.Fatalf("Failed to parse image name: %v", err)
}
// Get the authenticator from the default Docker keychain
auth, err := authn.DefaultKeychain.Resolve(ref.Context())
if err != nil {
log.Fatalf("Failed to get authenticator: %v", err)
}
// Push the image to the registry
switch image := image.(type) {
case *v1.Image:
if err := remote.Write(ref, *image, remote.WithAuth(auth)); err != nil {
return fmt.Errorf("failed to push image %s: %w", imageName, err)
}
case *v1.ImageIndex:
if err := remote.WriteIndex(ref, *image, remote.WithAuth(auth)); err != nil {
return fmt.Errorf("failed to push image index %s: %w", imageName, err)
}
default:
return fmt.Errorf("unknown image type: %T", image)
}
return nil
}
func SaveAsOCILayout(image any, path string) error {
// Save the image to the local filesystem
err := os.MkdirAll(path, os.FileMode(0744))
if err != nil {
return fmt.Errorf("failed to create directory: %w", err)
}
switch image := image.(type) {
case *v1.Image:
index := empty.Index
l, err := layout.Write(path, index)
if err != nil {
return fmt.Errorf("failed to create index: %w", err)
}
err = l.AppendImage(*image)
if err != nil {
return fmt.Errorf("failed to append image to index: %w", err)
}
case *v1.ImageIndex:
_, err := layout.Write(path, *image)
if err != nil {
return fmt.Errorf("failed to create index: %w", err)
}
default:
return fmt.Errorf("unknown image type: %T", image)
}
return nil
}

109
pkg/mirror/targets.go Normal file
View File

@@ -0,0 +1,109 @@
package mirror
import (
"fmt"
"path/filepath"
"strings"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/static"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/theupdateframework/go-tuf/v2/metadata"
)
// GetTufTargetMirrors returns a list of top-level target files as MirrorImages (image with tag)
func (m *TufMirror) GetTufTargetMirrors() ([]*MirrorImage, error) {
targetMirrors := []*MirrorImage{}
md := m.TufClient.GetMetadata()
// for each top-level target file, create an image with the target file as a layer
targets := md.Targets[metadata.TARGETS].Signed.Targets
for _, t := range targets {
// download target file
_, data, err := m.TufClient.DownloadTarget(t.Path, filepath.Join(m.tufPath, "download"))
if err != nil {
return nil, fmt.Errorf("failed to download target %s: %w", t.Path, err)
}
// create image with target file as layer
img := empty.Image
img = mutate.MediaType(img, types.OCIManifestSchema1)
img = mutate.ConfigMediaType(img, types.OCIConfigJSON)
// annotate layer
hash, ok := t.Hashes["sha256"]
if !ok {
return nil, fmt.Errorf("missing sha256 hash for target %s", t.Path)
}
name := strings.Join([]string{hash.String(), t.Path}, ".")
ann := map[string]string{tufFileAnnotation: name}
layer := mutate.Addendum{Layer: static.NewLayer(data, tufTargetMediaType), Annotations: ann}
img, err = mutate.Append(img, layer)
if err != nil {
return nil, fmt.Errorf("failed to append role layer to image: %w", err)
}
targetMirrors = append(targetMirrors, &MirrorImage{Image: &img, Tag: name})
}
return targetMirrors, nil
}
// GetDelegatedTargetMirrors returns a list of delegated target files as MirrorIndexes (image index with tag)
// each image in the index contains a delegated target file
func (m *TufMirror) GetDelegatedTargetMirrors() ([]*MirrorIndex, error) {
mirror := []*MirrorIndex{}
md := m.TufClient.GetMetadata()
// for each delegated role, create an image index with target files as images
roles := md.Targets[metadata.TARGETS].Signed.Delegations.Roles
for _, role := range roles {
// create an image index
index := v1.ImageIndex(empty.Index)
// get delegated targets metadata for role
roleMeta, err := m.TufClient.LoadDelegatedTargets(role.Name, metadata.TARGETS)
if err != nil {
return nil, fmt.Errorf("failed to load delegated targets metadata: %w", err)
}
// for each target file, create an image with the target file as a layer
for _, target := range roleMeta.Signed.Targets {
// download target file
_, data, err := m.TufClient.DownloadTarget(target.Path, filepath.Join(m.tufPath, "download"))
if err != nil {
return nil, fmt.Errorf("failed to download target %s: %w", target.Path, err)
}
// create image with target file as layer
img := empty.Image
img = mutate.MediaType(img, types.OCIManifestSchema1)
img = mutate.ConfigMediaType(img, types.OCIConfigJSON)
// annotate layer
hash, ok := target.Hashes["sha256"]
if !ok {
return nil, fmt.Errorf("missing sha256 hash for target %s", target.Path)
}
filename := filepath.Base(target.Path)
subdir, ok := strings.CutSuffix(target.Path, "/"+filename)
if !ok {
return nil, fmt.Errorf("failed to find target subdirectory [%s] in path: %s", subdir, target.Path)
}
name := strings.Join([]string{hash.String(), filename}, ".")
ann := map[string]string{tufFileAnnotation: name}
layer := mutate.Addendum{Layer: static.NewLayer(data, tufTargetMediaType), Annotations: ann}
img, err = mutate.Append(img, layer)
if err != nil {
return nil, fmt.Errorf("failed to append role layer to image: %w", err)
}
// append image to index with annotation
index = mutate.AppendManifests(index, mutate.IndexAddendum{
Add: img,
Descriptor: v1.Descriptor{
Annotations: map[string]string{
tufFileAnnotation: fmt.Sprintf("%s/%s", subdir, name),
},
},
})
}
mirror = append(mirror, &MirrorIndex{Index: &index, Tag: role.Name})
}
return mirror, nil
}

103
pkg/mirror/targets_test.go Normal file
View File

@@ -0,0 +1,103 @@
package mirror
import (
"encoding/json"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"github.com/docker/attest/internal/embed"
"github.com/docker/attest/internal/test"
"github.com/stretchr/testify/assert"
)
type Layer struct {
Annotations map[string]string `json:"annotations"`
Digest string `json:"digest"`
}
type Layers struct {
Layers []Layer `json:"layers"`
}
func TestGetTufTargetsMirror(t *testing.T) {
server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join("..", "..", "internal", "test", "testdata", "test-repo"))))
defer server.Close()
path := test.CreateTempDir(t, "", "tuf_temp")
m, err := NewTufMirror(embed.DevRoot, path, server.URL+"/metadata", server.URL+"/targets")
assert.Nil(t, err)
targets, err := m.GetTufTargetMirrors()
assert.Nil(t, err)
assert.Greater(t, len(targets), 0)
// check for image layer annotations
for _, target := range targets {
img := *target.Image
mf, err := img.RawManifest()
assert.Nil(t, err)
// unmarshal manifest with annotations
l := &Layers{}
err = json.Unmarshal(mf, l)
assert.Nil(t, err)
// check that layers are annotated
for _, layer := range l.Layers {
ann, ok := layer.Annotations[tufFileAnnotation]
assert.True(t, ok)
parts := strings.Split(ann, ".")
// <digest>.filename.json
assert.Equal(t, len(parts), 3)
}
}
}
func TestTargetDelegationMetadata(t *testing.T) {
server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join("..", "..", "internal", "test", "testdata", "test-repo"))))
defer server.Close()
path := test.CreateTempDir(t, "", "tuf_temp")
tm, err := NewTufMirror(embed.DevRoot, path, server.URL+"/metadata", server.URL+"/targets")
assert.Nil(t, err)
targets, err := tm.TufClient.LoadDelegatedTargets("test-role", "targets")
assert.Nil(t, err)
assert.Greater(t, len(targets.Signed.Targets), 0)
}
func TestGetDelegatedTargetMirrors(t *testing.T) {
server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join("..", "..", "internal", "test", "testdata", "test-repo"))))
defer server.Close()
path := test.CreateTempDir(t, "", "tuf_temp")
m, err := NewTufMirror(embed.DevRoot, path, server.URL+"/metadata", server.URL+"/targets")
assert.Nil(t, err)
mirrors, err := m.GetDelegatedTargetMirrors()
assert.Nil(t, err)
assert.Greater(t, len(mirrors), 0)
// check for index image annotations
for _, mirror := range mirrors {
idx := *mirror.Index
mf, err := idx.RawManifest()
assert.Nil(t, err)
// unmarshal manifest with annotations
l := &Layers{}
err = json.Unmarshal(mf, l)
assert.Nil(t, err)
// check that layers are annotated
for _, layer := range l.Layers {
ann, ok := layer.Annotations[tufFileAnnotation]
assert.True(t, ok)
parts := strings.Split(ann, ".")
// <subdir>/<digest>.filename.json
assert.Equal(t, len(parts), 3)
}
}
}

49
pkg/mirror/types.go Normal file
View File

@@ -0,0 +1,49 @@
package mirror
import (
"github.com/docker/attest/pkg/tuf"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/theupdateframework/go-tuf/v2/metadata"
)
const (
DefaultMetadataURL = "https://docker.github.io/tuf-staging/metadata"
DefaultTargetsURL = "https://docker.github.io/tuf-staging/targets"
tufMetadataMediaType = "application/vnd.tuf.metadata+json"
tufTargetMediaType = "application/vnd.tuf.target"
tufFileAnnotation = "tuf.io/filename"
)
type TufRole string
var TufRoles = []TufRole{metadata.ROOT, metadata.SNAPSHOT, metadata.TARGETS, metadata.TIMESTAMP}
type TufMetadata struct {
Root map[string][]byte
Snapshot map[string][]byte
Targets map[string][]byte
Timestamp []byte
}
type DelegatedTargetMetadata struct {
Name string
Version string
Data []byte
}
type MirrorImage struct {
Image *v1.Image
Tag string
}
type MirrorIndex struct {
Index *v1.ImageIndex
Tag string
}
type TufMirror struct {
TufClient *tuf.TufClient
tufPath string
metadataURL string
targetsURL string
}