feat: combine tuf code

This commit is contained in:
mrjoelkamp
2024-04-15 15:20:56 -05:00
parent 95295bc736
commit c1035c951e
56 changed files with 2660 additions and 0 deletions

196
pkg/mirror/metadata.go Normal file
View File

@@ -0,0 +1,196 @@
package mirror
import (
"fmt"
"strconv"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/static"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/theupdateframework/go-tuf/v2/metadata"
)
// -----------------
// TUF root metadata
// -----------------
// GetMetadataManifest returns an image with TUF root metadata as layers
func (m *TufMirror) GetMetadataManifest(metadataURL string) (*v1.Image, error) {
metadata, err := m.getTufMetadataMirror(metadataURL)
if err != nil {
return nil, fmt.Errorf("failed to get metadata: %w", err)
}
manifest, err := m.buildMetadataManifest(metadata)
if err != nil {
return nil, fmt.Errorf("failed to build metadata manifest: %w", err)
}
return manifest, nil
}
// getTufMetadataMirror returns a TufMetadata struct with TUF metadata as map of file names to bytes
func (m *TufMirror) getTufMetadataMirror(metadataURL string) (*TufMetadata, error) {
trustedMetadata := m.TufClient.GetMetadata()
rootMetadata := map[string][]byte{}
rootVersion := trustedMetadata.Root.Signed.Version
// get the previous versions of root metadata if any
if rootVersion != 1 {
var err error
rootMetadata, err = m.TufClient.GetPriorRoots(metadataURL)
if err != nil {
return nil, fmt.Errorf("failed to get prior root metadata: %w", err)
}
}
// get current root metadata
rootBytes, err := trustedMetadata.Root.ToBytes(false)
if err != nil {
return nil, fmt.Errorf("failed to get root metadata: %w", err)
}
rootMetadata[nameFromRole(metadata.ROOT, strconv.FormatInt(rootVersion, 10))] = rootBytes
snapshotBytes, err := trustedMetadata.Snapshot.ToBytes(false)
if err != nil {
return nil, fmt.Errorf("failed to get snapshot metadata: %w", err)
}
targetsBytes, err := trustedMetadata.Targets[metadata.TARGETS].ToBytes(false)
if err != nil {
return nil, fmt.Errorf("failed to get targets metadata: %w", err)
}
timestampBytes, err := trustedMetadata.Timestamp.ToBytes(false)
if err != nil {
return nil, fmt.Errorf("failed to get timestamp metadata: %w", err)
}
snapshotVersion := ""
targetsVersion := ""
if trustedMetadata.Root.Signed.ConsistentSnapshot {
snapshotVersion = strconv.FormatInt(trustedMetadata.Snapshot.Signed.Version, 10)
targetsVersion = strconv.FormatInt(trustedMetadata.Targets[metadata.TARGETS].Signed.Version, 10)
}
return &TufMetadata{
Root: rootMetadata,
Snapshot: map[string][]byte{nameFromRole(metadata.SNAPSHOT, snapshotVersion): snapshotBytes},
Targets: map[string][]byte{nameFromRole(metadata.TARGETS, targetsVersion): targetsBytes},
Timestamp: timestampBytes,
}, nil
}
// buildMetadataManifest returns an OCI image with TUF metadata as layers with annotations
func (m *TufMirror) buildMetadataManifest(metadata *TufMetadata) (*v1.Image, error) {
img := empty.Image
img = mutate.MediaType(img, types.OCIManifestSchema1)
img = mutate.ConfigMediaType(img, types.OCIConfigJSON)
for _, role := range TufRoles {
layers, err := m.makeRoleLayers(role, metadata)
if err != nil {
return nil, fmt.Errorf("failed to make role layer: %w", err)
}
img, err = mutate.Append(img, *layers...)
if err != nil {
return nil, fmt.Errorf("failed to append role layer to image: %w", err)
}
}
return &img, nil
}
// makeRoleLayers returns a list of layers for a given TUF role
func (m *TufMirror) makeRoleLayers(role TufRole, tufMetadata *TufMetadata) (*[]mutate.Addendum, error) {
layers := new([]mutate.Addendum)
ann := map[string]string{tufFileAnnotation: ""}
switch role {
case metadata.ROOT:
layers = m.annotatedMetaLayers(tufMetadata.Root)
case metadata.SNAPSHOT:
layers = m.annotatedMetaLayers(tufMetadata.Snapshot)
case metadata.TARGETS:
layers = m.annotatedMetaLayers(tufMetadata.Targets)
case metadata.TIMESTAMP:
ann[tufFileAnnotation] = fmt.Sprintf("%s.json", role)
*layers = append(*layers, mutate.Addendum{Layer: static.NewLayer(tufMetadata.Timestamp, tufMetadataMediaType), Annotations: ann})
default:
return nil, fmt.Errorf("unsupported TUF role: %s", role)
}
return layers, nil
}
// annotatedMetaLayers returns a list of layers with annotations for each TUF metadata file
func (m *TufMirror) annotatedMetaLayers(meta map[string][]byte) *[]mutate.Addendum {
layers := new([]mutate.Addendum)
for name, data := range meta {
ann := map[string]string{tufFileAnnotation: name}
*layers = append(*layers, mutate.Addendum{Layer: static.NewLayer(data, tufMetadataMediaType), Annotations: ann})
}
return layers
}
// ------------------------------
// TUF delegated targets metadata
// ------------------------------
// GetDelegatedMetadataMirrors returns a list of mirrors (image/tag pairs) for each delegated targets role metadata
func (m *TufMirror) GetDelegatedMetadataMirrors() ([]*MirrorImage, error) {
// get current delegated targets metadata
delegatedTargets, err := m.getDelegatedTargetsMetadata()
if err != nil {
return nil, fmt.Errorf("failed to get delegated targets metadata: %w", err)
}
mirror, err := m.buildDelegatedMetadataManifests(delegatedTargets)
if err != nil {
return nil, fmt.Errorf("failed to build delegated targets manifests: %w", err)
}
return mirror, nil
}
// getDelegatedTargetsMetadata returns delegated targets metadata as a list of DelegatedTargetMetadata (role name and data)
func (m *TufMirror) getDelegatedTargetsMetadata() (*[]DelegatedTargetMetadata, error) {
delegatedTargets := new([]DelegatedTargetMetadata)
md := m.TufClient.GetMetadata()
for _, role := range md.Targets[metadata.TARGETS].Signed.Delegations.Roles {
roleMetadata, err := m.TufClient.LoadDelegatedTargets(role.Name, metadata.TARGETS)
if err != nil {
return nil, fmt.Errorf("failed to get delegated role metadata: %w", err)
}
roleBytes, err := roleMetadata.ToBytes(false)
if err != nil {
return nil, fmt.Errorf("failed to get role %s metadata: %w", role.Name, err)
}
meta, ok := md.Snapshot.Signed.Meta[nameFromRole(role.Name, "")]
if !ok {
return nil, fmt.Errorf("failed to get role %s metadata: %w", role.Name, err)
}
// extract target metadata version in case of consistent snapshot naming
version := ""
if md.Root.Signed.ConsistentSnapshot {
version = strconv.FormatInt(meta.Version, 10)
}
*delegatedTargets = append(*delegatedTargets, DelegatedTargetMetadata{Name: role.Name, Version: version, Data: roleBytes})
}
return delegatedTargets, nil
}
// buildDelegatedMetadataManifests returns a list of mirrors (image/tag pairs) for each delegated target role metadata
func (m *TufMirror) buildDelegatedMetadataManifests(delegated *[]DelegatedTargetMetadata) ([]*MirrorImage, error) {
manifests := []*MirrorImage{}
for _, role := range *delegated {
img := empty.Image
img = mutate.MediaType(img, types.OCIManifestSchema1)
img = mutate.ConfigMediaType(img, types.OCIConfigJSON)
ann := map[string]string{tufFileAnnotation: nameFromRole(role.Name, role.Version)}
layer := mutate.Addendum{Layer: static.NewLayer(role.Data, tufMetadataMediaType), Annotations: ann}
img, err := mutate.Append(img, layer)
if err != nil {
return nil, fmt.Errorf("failed to append delegated targets layer to image: %w", err)
}
manifests = append(manifests, &MirrorImage{Image: &img, Tag: role.Name})
}
return manifests, nil
}
func nameFromRole(role, version string) string {
if version != "" {
return fmt.Sprintf("%s.%s.json", version, role)
}
return fmt.Sprintf("%s.json", role)
}

View File

@@ -0,0 +1,89 @@
package mirror
import (
"encoding/json"
"net/http"
"net/http/httptest"
"path/filepath"
"strconv"
"strings"
"testing"
"github.com/docker/attest/internal/embed"
"github.com/docker/attest/internal/test"
"github.com/stretchr/testify/assert"
"github.com/theupdateframework/go-tuf/v2/metadata"
)
func TestGetTufMetadataMirror(t *testing.T) {
server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join("..", "..", "internal", "test", "testdata", "test-repo"))))
defer server.Close()
path := test.CreateTempDir(t, "", "tuf_temp")
m, err := NewTufMirror(embed.DevRoot, path, server.URL+"/metadata", server.URL+"/targets")
assert.Nil(t, err)
tufMetadata, err := m.getTufMetadataMirror(server.URL + "/metadata")
assert.Nil(t, err)
// check that all roles are not empty
assert.Greater(t, len(tufMetadata.Root), 0)
assert.Greater(t, len(tufMetadata.Snapshot), 0)
assert.Greater(t, len(tufMetadata.Targets), 0)
assert.Greater(t, len(tufMetadata.Timestamp), 0)
}
func TestGetMetadataManifest(t *testing.T) {
server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join("..", "..", "internal", "test", "testdata", "test-repo"))))
defer server.Close()
path := test.CreateTempDir(t, "", "tuf_temp")
m, err := NewTufMirror(embed.DevRoot, path, server.URL+"/metadata", server.URL+"/targets")
assert.Nil(t, err)
img, err := m.GetMetadataManifest(server.URL + "/metadata")
assert.Nil(t, err)
assert.NotNil(t, img)
image := *img
mf, err := image.RawManifest()
assert.Nil(t, err)
type Annotations struct {
Annotations map[string]string `json:"annotations"`
}
type Layers struct {
Layers []Annotations `json:"layers"`
}
l := &Layers{}
err = json.Unmarshal(mf, l)
assert.Nil(t, err)
// check that layers are annotated and use consistent snapshot naming
for _, layer := range l.Layers {
ann, ok := layer.Annotations[tufFileAnnotation]
assert.True(t, ok)
// check for consistent snapshot version
parts := strings.Split(ann, ".")
if parts[0] == metadata.TIMESTAMP {
continue
}
_, err := strconv.Atoi(parts[0])
assert.Nil(t, err)
}
}
func TestGetDelegatedMetadataMirrors(t *testing.T) {
server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join("..", "..", "internal", "test", "testdata", "test-repo"))))
defer server.Close()
path := test.CreateTempDir(t, "", "tuf_temp")
m, err := NewTufMirror(embed.DevRoot, path, server.URL+"/metadata", server.URL+"/targets")
assert.Nil(t, err)
delegations, err := m.GetDelegatedMetadataMirrors()
assert.Nil(t, err)
assert.NotNil(t, delegations)
assert.Greater(t, len(delegations), 0)
}

82
pkg/mirror/mirror.go Normal file
View File

@@ -0,0 +1,82 @@
package mirror
import (
"fmt"
"log"
"os"
"github.com/docker/attest/internal/embed"
"github.com/docker/attest/pkg/tuf"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/layout"
"github.com/google/go-containerregistry/pkg/v1/remote"
)
func NewTufMirror(root []byte, tufPath, metadataURL, targetsURL string) (*TufMirror, error) {
if root == nil {
root = embed.DefaultRoot
}
tufClient, err := tuf.NewTufClient(root, tufPath, metadataURL, targetsURL)
if err != nil {
return nil, fmt.Errorf("failed to create TUF client: %w", err)
}
return &TufMirror{TufClient: tufClient, tufPath: tufPath, metadataURL: metadataURL, targetsURL: targetsURL}, nil
}
func PushToRegistry(image any, imageName string) error {
// Parse the image name
ref, err := name.ParseReference(imageName)
if err != nil {
log.Fatalf("Failed to parse image name: %v", err)
}
// Get the authenticator from the default Docker keychain
auth, err := authn.DefaultKeychain.Resolve(ref.Context())
if err != nil {
log.Fatalf("Failed to get authenticator: %v", err)
}
// Push the image to the registry
switch image := image.(type) {
case *v1.Image:
if err := remote.Write(ref, *image, remote.WithAuth(auth)); err != nil {
return fmt.Errorf("failed to push image %s: %w", imageName, err)
}
case *v1.ImageIndex:
if err := remote.WriteIndex(ref, *image, remote.WithAuth(auth)); err != nil {
return fmt.Errorf("failed to push image index %s: %w", imageName, err)
}
default:
return fmt.Errorf("unknown image type: %T", image)
}
return nil
}
func SaveAsOCILayout(image any, path string) error {
// Save the image to the local filesystem
err := os.MkdirAll(path, os.FileMode(0744))
if err != nil {
return fmt.Errorf("failed to create directory: %w", err)
}
switch image := image.(type) {
case *v1.Image:
index := empty.Index
l, err := layout.Write(path, index)
if err != nil {
return fmt.Errorf("failed to create index: %w", err)
}
err = l.AppendImage(*image)
if err != nil {
return fmt.Errorf("failed to append image to index: %w", err)
}
case *v1.ImageIndex:
_, err := layout.Write(path, *image)
if err != nil {
return fmt.Errorf("failed to create index: %w", err)
}
default:
return fmt.Errorf("unknown image type: %T", image)
}
return nil
}

109
pkg/mirror/targets.go Normal file
View File

@@ -0,0 +1,109 @@
package mirror
import (
"fmt"
"path/filepath"
"strings"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/static"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/theupdateframework/go-tuf/v2/metadata"
)
// GetTufTargetMirrors returns a list of top-level target files as MirrorImages (image with tag)
func (m *TufMirror) GetTufTargetMirrors() ([]*MirrorImage, error) {
targetMirrors := []*MirrorImage{}
md := m.TufClient.GetMetadata()
// for each top-level target file, create an image with the target file as a layer
targets := md.Targets[metadata.TARGETS].Signed.Targets
for _, t := range targets {
// download target file
_, data, err := m.TufClient.DownloadTarget(t.Path, filepath.Join(m.tufPath, "download"))
if err != nil {
return nil, fmt.Errorf("failed to download target %s: %w", t.Path, err)
}
// create image with target file as layer
img := empty.Image
img = mutate.MediaType(img, types.OCIManifestSchema1)
img = mutate.ConfigMediaType(img, types.OCIConfigJSON)
// annotate layer
hash, ok := t.Hashes["sha256"]
if !ok {
return nil, fmt.Errorf("missing sha256 hash for target %s", t.Path)
}
name := strings.Join([]string{hash.String(), t.Path}, ".")
ann := map[string]string{tufFileAnnotation: name}
layer := mutate.Addendum{Layer: static.NewLayer(data, tufTargetMediaType), Annotations: ann}
img, err = mutate.Append(img, layer)
if err != nil {
return nil, fmt.Errorf("failed to append role layer to image: %w", err)
}
targetMirrors = append(targetMirrors, &MirrorImage{Image: &img, Tag: name})
}
return targetMirrors, nil
}
// GetDelegatedTargetMirrors returns a list of delegated target files as MirrorIndexes (image index with tag)
// each image in the index contains a delegated target file
func (m *TufMirror) GetDelegatedTargetMirrors() ([]*MirrorIndex, error) {
mirror := []*MirrorIndex{}
md := m.TufClient.GetMetadata()
// for each delegated role, create an image index with target files as images
roles := md.Targets[metadata.TARGETS].Signed.Delegations.Roles
for _, role := range roles {
// create an image index
index := v1.ImageIndex(empty.Index)
// get delegated targets metadata for role
roleMeta, err := m.TufClient.LoadDelegatedTargets(role.Name, metadata.TARGETS)
if err != nil {
return nil, fmt.Errorf("failed to load delegated targets metadata: %w", err)
}
// for each target file, create an image with the target file as a layer
for _, target := range roleMeta.Signed.Targets {
// download target file
_, data, err := m.TufClient.DownloadTarget(target.Path, filepath.Join(m.tufPath, "download"))
if err != nil {
return nil, fmt.Errorf("failed to download target %s: %w", target.Path, err)
}
// create image with target file as layer
img := empty.Image
img = mutate.MediaType(img, types.OCIManifestSchema1)
img = mutate.ConfigMediaType(img, types.OCIConfigJSON)
// annotate layer
hash, ok := target.Hashes["sha256"]
if !ok {
return nil, fmt.Errorf("missing sha256 hash for target %s", target.Path)
}
filename := filepath.Base(target.Path)
subdir, ok := strings.CutSuffix(target.Path, "/"+filename)
if !ok {
return nil, fmt.Errorf("failed to find target subdirectory [%s] in path: %s", subdir, target.Path)
}
name := strings.Join([]string{hash.String(), filename}, ".")
ann := map[string]string{tufFileAnnotation: name}
layer := mutate.Addendum{Layer: static.NewLayer(data, tufTargetMediaType), Annotations: ann}
img, err = mutate.Append(img, layer)
if err != nil {
return nil, fmt.Errorf("failed to append role layer to image: %w", err)
}
// append image to index with annotation
index = mutate.AppendManifests(index, mutate.IndexAddendum{
Add: img,
Descriptor: v1.Descriptor{
Annotations: map[string]string{
tufFileAnnotation: fmt.Sprintf("%s/%s", subdir, name),
},
},
})
}
mirror = append(mirror, &MirrorIndex{Index: &index, Tag: role.Name})
}
return mirror, nil
}

103
pkg/mirror/targets_test.go Normal file
View File

@@ -0,0 +1,103 @@
package mirror
import (
"encoding/json"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"github.com/docker/attest/internal/embed"
"github.com/docker/attest/internal/test"
"github.com/stretchr/testify/assert"
)
type Layer struct {
Annotations map[string]string `json:"annotations"`
Digest string `json:"digest"`
}
type Layers struct {
Layers []Layer `json:"layers"`
}
func TestGetTufTargetsMirror(t *testing.T) {
server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join("..", "..", "internal", "test", "testdata", "test-repo"))))
defer server.Close()
path := test.CreateTempDir(t, "", "tuf_temp")
m, err := NewTufMirror(embed.DevRoot, path, server.URL+"/metadata", server.URL+"/targets")
assert.Nil(t, err)
targets, err := m.GetTufTargetMirrors()
assert.Nil(t, err)
assert.Greater(t, len(targets), 0)
// check for image layer annotations
for _, target := range targets {
img := *target.Image
mf, err := img.RawManifest()
assert.Nil(t, err)
// unmarshal manifest with annotations
l := &Layers{}
err = json.Unmarshal(mf, l)
assert.Nil(t, err)
// check that layers are annotated
for _, layer := range l.Layers {
ann, ok := layer.Annotations[tufFileAnnotation]
assert.True(t, ok)
parts := strings.Split(ann, ".")
// <digest>.filename.json
assert.Equal(t, len(parts), 3)
}
}
}
func TestTargetDelegationMetadata(t *testing.T) {
server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join("..", "..", "internal", "test", "testdata", "test-repo"))))
defer server.Close()
path := test.CreateTempDir(t, "", "tuf_temp")
tm, err := NewTufMirror(embed.DevRoot, path, server.URL+"/metadata", server.URL+"/targets")
assert.Nil(t, err)
targets, err := tm.TufClient.LoadDelegatedTargets("test-role", "targets")
assert.Nil(t, err)
assert.Greater(t, len(targets.Signed.Targets), 0)
}
func TestGetDelegatedTargetMirrors(t *testing.T) {
server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join("..", "..", "internal", "test", "testdata", "test-repo"))))
defer server.Close()
path := test.CreateTempDir(t, "", "tuf_temp")
m, err := NewTufMirror(embed.DevRoot, path, server.URL+"/metadata", server.URL+"/targets")
assert.Nil(t, err)
mirrors, err := m.GetDelegatedTargetMirrors()
assert.Nil(t, err)
assert.Greater(t, len(mirrors), 0)
// check for index image annotations
for _, mirror := range mirrors {
idx := *mirror.Index
mf, err := idx.RawManifest()
assert.Nil(t, err)
// unmarshal manifest with annotations
l := &Layers{}
err = json.Unmarshal(mf, l)
assert.Nil(t, err)
// check that layers are annotated
for _, layer := range l.Layers {
ann, ok := layer.Annotations[tufFileAnnotation]
assert.True(t, ok)
parts := strings.Split(ann, ".")
// <subdir>/<digest>.filename.json
assert.Equal(t, len(parts), 3)
}
}
}

49
pkg/mirror/types.go Normal file
View File

@@ -0,0 +1,49 @@
package mirror
import (
"github.com/docker/attest/pkg/tuf"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/theupdateframework/go-tuf/v2/metadata"
)
const (
DefaultMetadataURL = "https://docker.github.io/tuf-staging/metadata"
DefaultTargetsURL = "https://docker.github.io/tuf-staging/targets"
tufMetadataMediaType = "application/vnd.tuf.metadata+json"
tufTargetMediaType = "application/vnd.tuf.target"
tufFileAnnotation = "tuf.io/filename"
)
type TufRole string
var TufRoles = []TufRole{metadata.ROOT, metadata.SNAPSHOT, metadata.TARGETS, metadata.TIMESTAMP}
type TufMetadata struct {
Root map[string][]byte
Snapshot map[string][]byte
Targets map[string][]byte
Timestamp []byte
}
type DelegatedTargetMetadata struct {
Name string
Version string
Data []byte
}
type MirrorImage struct {
Image *v1.Image
Tag string
}
type MirrorIndex struct {
Index *v1.ImageIndex
Tag string
}
type TufMirror struct {
TufClient *tuf.TufClient
tufPath string
metadataURL string
targetsURL string
}

60
pkg/tuf/mock.go Normal file
View File

@@ -0,0 +1,60 @@
package tuf
import (
"io"
"os"
"path/filepath"
)
type mockTufClient struct {
srcPath string
dstPath string
}
func NewMockTufClient(srcPath string, dstPath string) *mockTufClient {
if srcPath == "" {
panic("srcPath must be set")
}
if dstPath == "" {
panic("dstPath must be set")
}
return &mockTufClient{
srcPath: srcPath,
dstPath: dstPath,
}
}
func (dc *mockTufClient) DownloadTarget(target string, filePath string) (actualFilePath string, data []byte, err error) {
src, err := os.Open(filepath.Join(dc.srcPath, target))
if err != nil {
return "", nil, err
}
defer src.Close()
var dstFilePath string
if filePath == "" {
dstFilePath = filepath.Join(dc.dstPath, filepath.FromSlash(target))
} else {
dstFilePath = filePath
}
err = os.MkdirAll(filepath.Dir(dstFilePath), 0755)
if err != nil {
return "", nil, err
}
dst, err := os.Create(dstFilePath)
if err != nil {
return "", nil, err
}
defer dst.Close()
// reading from tee will read from src and write to dst at the same time
tee := io.TeeReader(src, dst)
b, err := io.ReadAll(tee)
if err != nil {
return "", nil, err
}
return dstFilePath, b, nil
}

306
pkg/tuf/registry.go Normal file
View File

@@ -0,0 +1,306 @@
package tuf
import (
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"path"
"strings"
"time"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/crane"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/theupdateframework/go-tuf/v2/metadata"
)
const (
TufFileNameAnnotation = "tuf.io/filename"
)
type TufRole string
var TufRoles = []TufRole{metadata.ROOT, metadata.SNAPSHOT, metadata.TARGETS, metadata.TIMESTAMP}
// RegistryFetcher implements Fetcher
type RegistryFetcher struct {
httpUserAgent string
metadataRepo string
metadataTag string
targetsRepo string
cache *ImageCache
timeout time.Duration
}
type ImageCache struct {
cache map[string][]byte
}
func NewImageCache() *ImageCache {
return &ImageCache{
cache: make(map[string][]byte),
}
}
// Get image from cache
func (c *ImageCache) Get(imgRef string) ([]byte, bool) {
img, found := c.cache[imgRef]
return img, found
}
// Add image to cache
func (c *ImageCache) Put(imgRef string, img []byte) {
c.cache[imgRef] = img
}
type Layer struct {
Annotations map[string]string `json:"annotations"`
Digest string `json:"digest"`
}
type Layers struct {
Layers []Layer `json:"layers"`
Manifests []Layer `json:"manifests"`
MediaType string `json:"mediaType"`
}
func NewRegistryFetcher(metadataRepo, metadataTag, targetsRepo string) *RegistryFetcher {
return &RegistryFetcher{
metadataRepo: metadataRepo,
metadataTag: metadataTag,
targetsRepo: targetsRepo,
cache: NewImageCache(),
}
}
// DownloadFile downloads a file from an OCI registry, errors out if it failed,
// its length is larger than maxLength or the timeout is reached.
func (d *RegistryFetcher) DownloadFile(urlPath string, maxLength int64, timeout time.Duration) ([]byte, error) {
d.timeout = timeout
imgRef, fileName, err := d.parseImgRef(urlPath)
if err != nil {
return nil, err
}
// Get manifest for image or index
mf, err := d.getManifest(imgRef)
if err != nil {
return nil, err
}
// Search image/index manifest for file
hash, err := d.findFileInManifest(mf, fileName)
if err != nil {
// TODO - refactor Fetcher interface for not found error handling? (requires go-tuf change)
return nil, &metadata.ErrDownloadHTTP{StatusCode: http.StatusNotFound}
}
// Get file from layer
parts := strings.Split(imgRef, ":")
switch len(parts) {
// default host port
case 2:
return d.pullFileLayer(fmt.Sprintf("%s@%s", parts[0], *hash), maxLength)
// custom host port
case 3:
return d.pullFileLayer(fmt.Sprintf("%s:%s@%s", parts[0], parts[1], *hash), maxLength)
default:
return nil, fmt.Errorf("invalid image reference: %s", imgRef)
}
}
// getManifest returns the manifest for an image or index
func (d *RegistryFetcher) getManifest(ref string) ([]byte, error) {
// Pull image manifest
var err error
var found bool
var mf []byte
// Check cache for manifest and only pull if not found
if mf, found = d.cache.Get(ref); !found {
mf, err = crane.Manifest(ref,
crane.WithUserAgent(d.httpUserAgent),
crane.WithTransport(transportWithTimeout(d.timeout)),
crane.WithAuth(authn.Anonymous),
crane.WithAuthFromKeychain(authn.DefaultKeychain))
if err != nil {
return nil, err
}
// Cache the manifest
d.cache.Put(ref, mf)
}
return mf, nil
}
// pullFileLayer pulls a layer for an image or index and returns its data
func (d *RegistryFetcher) pullFileLayer(ref string, maxLength int64) ([]byte, error) {
var data []byte
var found bool
// Check cache for layer and only pull if not found
if data, found = d.cache.Get(ref); !found {
layer, err := crane.PullLayer(ref,
crane.WithUserAgent(d.httpUserAgent),
crane.WithTransport(transportWithTimeout(d.timeout)),
crane.WithAuth(authn.Anonymous),
crane.WithAuthFromKeychain(authn.DefaultKeychain))
if err != nil {
return nil, err
}
data, err = getDataFromLayer(layer, maxLength)
if err != nil {
return nil, err
}
// Cache the layer
d.cache.Put(ref, data)
}
return data, nil
}
// getDataFromLayer returns the data from a layer in an image
func getDataFromLayer(fileLayer v1.Layer, maxLength int64) ([]byte, error) {
length, err := fileLayer.Size()
if err != nil {
return nil, err
}
// Error if the reported size is greater than what is expected.
if length > maxLength {
return nil, &metadata.ErrDownloadLengthMismatch{Msg: fmt.Sprintf("download failed, length %d is larger than expected %d", length, maxLength)}
}
content, err := fileLayer.Uncompressed()
if err != nil {
return nil, err
}
data, err := io.ReadAll(io.LimitReader(content, maxLength+1))
if err != nil {
return nil, err
}
// Error if the reported size is greater than what is expected.
length = int64(len(data))
if length > maxLength {
return nil, &metadata.ErrDownloadLengthMismatch{Msg: fmt.Sprintf("download failed, length %d is larger than expected %d", length, maxLength)}
}
return data, nil
}
// parseImgRef maintains the Fetcher interface by parsing a URL path to an image reference and file name
func (d *RegistryFetcher) parseImgRef(urlPath string) (imgRef, fileName string, err error) {
// Check if repo is target or metadata
if strings.Contains(urlPath, d.targetsRepo) {
// determine if the target path contains subdirectories and set image name accordingly
// <repo>/<filename> -> image = <repo>:<filename>, layer = <filename>
// <repo>/<subdir>/<filename> -> index = <repo>:<subdir> , image = <filename> -> layer = <filename>
target := strings.TrimPrefix(urlPath, d.targetsRepo+"/")
subdir, name, found := strings.Cut(target, "/")
if found {
return fmt.Sprintf("%s:%s", d.targetsRepo, subdir), fmt.Sprintf("%s/%s", subdir, name), nil
}
return fmt.Sprintf("%s:%s", d.targetsRepo, target), target, nil
} else if strings.Contains(urlPath, d.metadataRepo) {
// build the metadata image name
// determine if role is a delegated role and set the tag accordingly
fileName = path.Base(urlPath)
role := roleFromConsistentName(fileName)
// if the role is a delegated role use the role name as the tag for imgRef
if isDelegatedRole(role) {
return fmt.Sprintf("%s:%s", d.metadataRepo, role), fileName, nil
}
return fmt.Sprintf("%s:%s", d.metadataRepo, d.metadataTag), fileName, nil
} else {
return "", "", fmt.Errorf("urlPath: %s must be in metadata or targets repo", urlPath)
}
}
// findFileInManifest searches the image or index manifest for a file with the given name and returns its digest
func (d *RegistryFetcher) findFileInManifest(mf []byte, name string) (*v1.Hash, error) {
var index bool
// unmarshal manifest with annotations
l := &Layers{}
err := json.Unmarshal(mf, l)
if err != nil {
return nil, err
}
// determine image or index manifest
var layers []Layer
if l.MediaType == string(types.OCIImageIndex) {
layers = l.Manifests
index = true
} else if l.MediaType == string(types.OCIManifestSchema1) {
layers = l.Layers
index = false
} else {
return nil, fmt.Errorf("invalid manifest media type: %s", l.MediaType)
}
// find annotation with file name
var digest string
for _, layer := range layers {
if layer.Annotations[TufFileNameAnnotation] == name {
digest = layer.Digest
break
}
}
if digest == "" {
return nil, fmt.Errorf("file %s not found in image", name)
}
// return layer digest as v1.Hash
hash := new(v1.Hash)
*hash, err = v1.NewHash(digest)
if err != nil {
return nil, err
}
// if index manifest pull image to get file layer
if index {
mf, err := d.getManifest(fmt.Sprintf("%s@%s", d.targetsRepo, *hash))
if err != nil {
return nil, err
}
parts := strings.Split(name, "/")
return d.findFileInManifest(mf, parts[len(parts)-1])
}
return hash, nil
}
// transportWithTimeout returns a http.RoundTripper with a specified timeout
func transportWithTimeout(timeout time.Duration) http.RoundTripper {
// transport is based on go-containerregistry remote.DefaultTransport
// with modifications to include a specified timeout
return &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: timeout,
KeepAlive: timeout,
}).DialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
MaxIdleConnsPerHost: 50,
}
}
// isDelegatedRole returns true if the role is a delegated role
func isDelegatedRole(role string) bool {
for _, r := range TufRoles {
if role == string(r) {
return false // role is not a delegated role
}
}
return true // role is a delegated role
}
// roleFromConsistentName returns the role name from a consistent snapshot file name
func roleFromConsistentName(filename string) string {
name := strings.TrimSuffix(filename, ".json")
role := strings.Split(name, ".")
if len(role) > 1 {
return role[1]
}
return role[0]
}

446
pkg/tuf/registry_test.go Normal file
View File

@@ -0,0 +1,446 @@
package tuf
import (
"context"
"fmt"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"github.com/docker/attest/internal/embed"
"github.com/docker/attest/internal/test"
"github.com/docker/attest/internal/util"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/crane"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/layout"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/static"
"github.com/google/go-containerregistry/pkg/v1/types"
"github.com/stretchr/testify/assert"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/registry"
"github.com/theupdateframework/go-tuf/v2/metadata"
"github.com/theupdateframework/go-tuf/v2/metadata/config"
"github.com/theupdateframework/go-tuf/v2/metadata/updater"
)
const (
tufTargetMediaType = "application/vnd.tuf.target"
)
func TestRegistryFetcher(t *testing.T) {
// run local registry
registry, regAddr := RunTestRegistry(t)
defer func() {
if err := registry.Terminate(context.Background()); err != nil {
t.Fatalf("failed to terminate container: %s", err) // nolint:gocritic
}
}()
LoadRegistryTestData(t, regAddr, OciTufTestDataPath)
metadataRepo := regAddr.Host + "/tuf-metadata"
metadataImgTag := "latest"
targetsRepo := regAddr.Host + "/tuf-targets"
targetFile := "test.txt"
delegatedRole := "test-role"
dir := test.CreateTempDir(t, "", "tuf_temp")
delegatedDir := test.CreateTempDir(t, dir, delegatedRole)
delegatedTargetFile := fmt.Sprintf("%s/%s", delegatedRole, targetFile)
cfg, err := config.New(metadataRepo, embed.DevRoot)
assert.NoError(t, err)
cfg.Fetcher = NewRegistryFetcher(metadataRepo, metadataImgTag, targetsRepo)
cfg.LocalMetadataDir = dir
cfg.LocalTargetsDir = dir
cfg.RemoteTargetsURL = targetsRepo
// create a new Updater instance
up, err := updater.New(cfg)
assert.NoError(t, err)
// refresh the metadata
err = up.Refresh()
assert.NoError(t, err)
// download top-level target
targetInfo, err := up.GetTargetInfo(targetFile)
assert.NoError(t, err)
_, _, err = up.DownloadTarget(targetInfo, filepath.Join(dir, targetInfo.Path), "")
assert.NoError(t, err)
// download delegated target
targetInfo, err = up.GetTargetInfo(delegatedTargetFile)
assert.NoError(t, err)
_, _, err = up.DownloadTarget(targetInfo, filepath.Join(delegatedDir, targetFile), "")
assert.NoError(t, err)
}
func TestRoleFromConsistentName(t *testing.T) {
testCases := []struct {
name string
expected string
}{
{"root.json", metadata.ROOT},
{"1.root.json", metadata.ROOT},
{"targets.json", metadata.TARGETS},
{"63.targets.json", metadata.TARGETS},
{"timestamp", metadata.TIMESTAMP},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.expected, roleFromConsistentName(tc.name))
})
}
}
func TestIsDelegatedRole(t *testing.T) {
testCases := []struct {
name string
expected bool
}{
{metadata.ROOT, false},
{metadata.TARGETS, false},
{metadata.TIMESTAMP, false},
{"doi", true},
{"test", true},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.expected, isDelegatedRole(tc.name))
})
}
}
func TestFindFileInManifest(t *testing.T) {
// make test image manifest
file := "test.json"
data := []byte("test")
hash := v1.Hash{Algorithm: "sha256", Hex: util.HexHashBytes(data)}
img := empty.Image
img = mutate.MediaType(img, types.OCIManifestSchema1)
img = mutate.ConfigMediaType(img, types.OCIConfigJSON)
// add test layer
name := strings.Join([]string{hash.Hex, file}, ".")
ann := map[string]string{TufFileNameAnnotation: name}
layer := mutate.Addendum{Layer: static.NewLayer(data, tufTargetMediaType), Annotations: ann}
img, err := mutate.Append(img, layer)
assert.NoError(t, err)
image_manifest, err := img.RawManifest()
assert.NoError(t, err)
// make test index manifest
idx := v1.ImageIndex(empty.Index)
assert.NoError(t, err)
// append image to index with annotation
idx = mutate.AppendManifests(idx, mutate.IndexAddendum{
Add: img,
Descriptor: v1.Descriptor{
Annotations: map[string]string{
TufFileNameAnnotation: name,
},
},
})
index_manifest, err := idx.RawManifest()
assert.NoError(t, err)
// cache image layer
targetsRepo := "test/tuf-targets"
d := &RegistryFetcher{
cache: NewImageCache(),
targetsRepo: targetsRepo,
}
imgHash, err := img.Digest()
assert.NoError(t, err)
d.cache.Put(fmt.Sprintf("%s@%s", targetsRepo, imgHash.String()), image_manifest)
testCases := []struct {
name string
manifest []byte
file string
expected string
}{
{"consistent filename image", image_manifest, fmt.Sprintf("%s.%s", hash.Hex, file), hash.Hex},
{"filename image", image_manifest, file, ""},
{"consistent filename index", index_manifest, fmt.Sprintf("%s.%s", hash.Hex, file), hash.Hex},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
digest, err := d.findFileInManifest(tc.manifest, tc.file)
if tc.expected == "" {
assert.Error(t, err)
return
}
assert.NoError(t, err)
assert.Equal(t, tc.expected, digest.Hex)
})
}
}
func TestParseImgRef(t *testing.T) {
metadataRepo := "test/tuf-metadata"
metadataTag := "latest"
targetsRepo := "test/tuf-targets"
delegatedRole := "test-role"
testCases := []struct {
name string
ref string
expectedRef string
expectedFile string
}{
{"top-level metadata", fmt.Sprintf("%s/2.root.json", metadataRepo), fmt.Sprintf("%s:%s", metadataRepo, metadataTag), "2.root.json"},
{"delegated metadata", fmt.Sprintf("%s/%s/5.test-role.json", metadataRepo, delegatedRole), fmt.Sprintf("%s:%s", metadataRepo, delegatedRole), "5.test-role.json"},
{"top-level target", fmt.Sprintf("%s/policy.yaml", targetsRepo), fmt.Sprintf("%s:policy.yaml", targetsRepo), "policy.yaml"},
{"delegated target", fmt.Sprintf("%s/%s/policy.yaml", targetsRepo, delegatedRole), fmt.Sprintf("%s:%s", targetsRepo, delegatedRole), fmt.Sprintf("%s/policy.yaml", delegatedRole)},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
d := &RegistryFetcher{
metadataRepo: metadataRepo,
metadataTag: "latest",
targetsRepo: targetsRepo,
}
imgRef, file, err := d.parseImgRef(tc.ref)
assert.NoError(t, err)
assert.Equal(t, tc.expectedRef, imgRef)
assert.Equal(t, tc.expectedFile, file)
})
}
}
func TestGetDataFromLayer(t *testing.T) {
data := []byte("test")
layer := static.NewLayer(data, tufTargetMediaType)
testCases := []struct {
name string
layer v1.Layer
max int64
expected []byte
}{
{"valid length", layer, int64(len(data)), data},
{"invalid length", layer, int64(len(data) - 1), nil},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
data, err := getDataFromLayer(tc.layer, tc.max)
if tc.expected == nil {
assert.Error(t, err)
return
}
assert.NoError(t, err)
assert.Equal(t, tc.expected, data)
})
}
}
func TestPullFileLayer(t *testing.T) {
// run local registry
registry, url := RunTestRegistry(t)
defer func() {
if err := registry.Terminate(context.Background()); err != nil {
t.Fatalf("failed to terminate container: %s", err) // nolint:gocritic
}
}()
// make test layer
repo := "tuf-metadata"
data := []byte("test")
testLayer := static.NewLayer(data, tufTargetMediaType)
hash, err := testLayer.Digest()
assert.NoError(t, err)
layerRef := fmt.Sprintf("%s/%s@%s", url.Host, repo, hash.String())
// cache test layer
d := &RegistryFetcher{
cache: NewImageCache(),
}
d.cache.Put(layerRef, data)
// push uncached image layer to local registry
uncachedData := []byte("uncached")
uncachedTestLayer := static.NewLayer(uncachedData, tufTargetMediaType)
uncachedHash, err := uncachedTestLayer.Digest()
assert.NoError(t, err)
uncachedLayerRef := fmt.Sprintf("%s/%s@%s", url.Host, repo, uncachedHash.String())
img := empty.Image
img, err = mutate.Append(img, mutate.Addendum{Layer: uncachedTestLayer})
assert.NoError(t, err)
err = crane.Push(img, fmt.Sprintf("%s/%s", url.Host, fmt.Sprintf("%s:latest", repo)))
assert.NoError(t, err)
testCases := []struct {
name string
ref string
maxLength int
expected []byte
}{
{"cached layer", layerRef, len(data), data},
{"uncached layer", uncachedLayerRef, len(uncachedData), uncachedData},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
layer, err := d.pullFileLayer(tc.ref, int64(tc.maxLength))
if tc.expected == nil {
assert.Error(t, err)
return
}
assert.NoError(t, err)
assert.Greater(t, len(layer), 0)
})
}
}
func TestGetManifest(t *testing.T) {
// run local registry
registry, url := RunTestRegistry(t)
defer func() {
if err := registry.Terminate(context.Background()); err != nil {
t.Fatalf("failed to terminate container: %s", err) // nolint:gocritic
}
}()
// make test manifest
repo := "tuf-metadata"
img := empty.Image
img = mutate.MediaType(img, types.OCIManifestSchema1)
img = mutate.ConfigMediaType(img, types.OCIConfigJSON)
imgRef := fmt.Sprintf("%s/%s:latest", url.Host, repo)
// cache test manifest
d := &RegistryFetcher{
cache: NewImageCache(),
}
mf, err := img.RawManifest()
assert.NoError(t, err)
d.cache.Put(imgRef, mf)
// push test image to local registry
unchachedImgRef := fmt.Sprintf("%s/%s:unchached", url.Host, repo)
err = crane.Push(img, unchachedImgRef)
assert.NoError(t, err)
testCases := []struct {
name string
ref string
expected []byte
}{
{"cached image manifest", imgRef, mf},
{"uncached image manifest", unchachedImgRef, mf},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
manifest, err := d.getManifest(tc.ref)
assert.NoError(t, err)
assert.Equal(t, tc.expected, manifest)
})
}
}
// RunTestRegistry starts a registry testcontainer for TUF on OCI testdata
func RunTestRegistry(t *testing.T) (*registry.RegistryContainer, *url.URL) {
registryContainer, err := registry.RunContainer(context.Background(), testcontainers.WithImage("registry:2.8.3"))
if err != nil {
t.Fatalf("failed to start container: %s", err)
}
httpAddress, err := registryContainer.Address(context.Background())
if err != nil {
t.Fatalf("failed to get container address: %s", err)
}
addr, err := url.Parse(httpAddress)
if err != nil {
t.Fatalf("failed to parse container address: %s", err)
}
if addr.Hostname() == "127.0.0.1" {
addr.Host = "localhost:" + addr.Port()
}
return registryContainer, addr
}
// LoadRegistryTestData pushes TUF metadata and targets to an OCI registry
func LoadRegistryTestData(t *testing.T, registry *url.URL, path string) {
// push tuf metadata and targets to local registry
METADATA_REPO := "tuf-metadata"
METADATA_TAG := "latest"
TARGETS_REPO := "tuf-targets"
DELEGATED_ROLE := "test-role"
// push top-level metadata -> metadata:latest
err := LoadMetadata(filepath.Join(path, "metadata"), registry.Host, METADATA_REPO, METADATA_TAG)
if err != nil {
t.Fatal(err)
}
// push delegated metadata -> metadata:<DELEGATED_ROLE>
err = LoadMetadata(filepath.Join(path, "metadata", DELEGATED_ROLE), registry.Host, METADATA_REPO, DELEGATED_ROLE)
if err != nil {
t.Fatal(err)
}
// push targets -> targets:<HASH>.<FILE>.ext (image) or targets:<DELEGATED ROLE> <index)
targetDirs, err := os.ReadDir(filepath.Join(path, "targets"))
if err != nil {
t.Fatal(err)
}
for _, dir := range targetDirs {
if !dir.IsDir() {
continue
}
tIdx, err := layout.ImageIndexFromPath(filepath.Join(path, "targets", dir.Name()))
if err != nil {
t.Fatal(err)
}
ref, err := name.ParseReference(fmt.Sprintf("%s/%s:%s", registry.Host, TARGETS_REPO, dir.Name()))
if err != nil {
t.Fatal(err)
}
mf, err := tIdx.IndexManifest()
if err != nil {
t.Fatal(err)
}
if len(mf.Manifests) == 1 {
// top-level target
img, err := tIdx.Image(mf.Manifests[0].Digest)
if err != nil {
t.Fatal(err)
}
err = remote.Write(ref, img, remote.WithAuthFromKeychain(authn.DefaultKeychain))
if err != nil {
t.Fatal(err)
}
} else if len(mf.Manifests) > 1 {
// delegated target
err = remote.WriteIndex(ref, tIdx, remote.WithAuthFromKeychain(authn.DefaultKeychain))
if err != nil {
t.Fatal(err)
}
} else {
t.Fatal("no manifests found")
}
}
}
// LoadMetadata loads TUF metadata from a local path and pushes to a registry
func LoadMetadata(path, host, repo, tag string) error {
mIdx, err := layout.ImageIndexFromPath(path)
if err != nil {
return err
}
ref, err := name.ParseReference(fmt.Sprintf("%s/%s:%s", host, repo, tag))
if err != nil {
return err
}
mf, err := mIdx.IndexManifest()
if err != nil {
return err
}
img, err := mIdx.Image(mf.Manifests[0].Digest)
if err != nil {
return err
}
return remote.Write(ref, img, remote.WithAuthFromKeychain(authn.DefaultKeychain))
}

216
pkg/tuf/tuf.go Normal file
View File

@@ -0,0 +1,216 @@
package tuf
import (
"errors"
"fmt"
"io/fs"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/docker/attest/internal/util"
"github.com/theupdateframework/go-tuf/v2/metadata"
"github.com/theupdateframework/go-tuf/v2/metadata/config"
"github.com/theupdateframework/go-tuf/v2/metadata/fetcher"
"github.com/theupdateframework/go-tuf/v2/metadata/trustedmetadata"
"github.com/theupdateframework/go-tuf/v2/metadata/updater"
)
type TufSource string
const (
HttpSource TufSource = "http"
OciSource TufSource = "oci"
)
type TUFClient interface {
DownloadTarget(target, filePath string) (actualFilePath string, data []byte, err error)
}
type TufClient struct {
updater *updater.Updater
cfg *config.UpdaterConfig
}
// NewTufClient creates a new TUF client
func NewTufClient(initialRoot []byte, tufPath, metadataSource, targetsSource string) (*TufClient, error) {
var tufSource TufSource
if strings.HasPrefix(metadataSource, "https://") || strings.HasPrefix(metadataSource, "http://") {
tufSource = HttpSource
} else {
tufSource = OciSource
}
tufRootDigest := util.HexHashBytes(initialRoot)
// create a directory for each initial root.json
metadataPath := filepath.Join(tufPath, tufRootDigest)
err := os.MkdirAll(metadataPath, 0755)
if err != nil {
return nil, fmt.Errorf("failed to create directory '%s': %w", metadataPath, err)
}
rootFile := filepath.Join(metadataPath, "root.json")
var rootBytes []byte
rootBytes, err = os.ReadFile(rootFile)
if err != nil {
if !errors.Is(err, fs.ErrNotExist) {
return nil, fmt.Errorf("failed to read root.json: %w", err)
}
// write the root.json file to the metadata directory
err = os.WriteFile(rootFile, initialRoot, 0644)
if err != nil {
return nil, fmt.Errorf("Failed to write root.json %w", err)
}
rootBytes = initialRoot
}
// create updater configuration
cfg, err := config.New(metadataSource, rootBytes) // default config
if err != nil {
return nil, fmt.Errorf("failed to create TUF updater configuration: %w", err)
}
cfg.LocalMetadataDir = metadataPath
cfg.LocalTargetsDir = filepath.Join(metadataPath, "download")
cfg.RemoteTargetsURL = targetsSource
if tufSource == OciSource {
metadataRepo, metadataTag, found := strings.Cut(metadataSource, ":")
if !found {
fmt.Printf("metadata tag not found in URL, using latest\n")
metadataTag = "latest"
}
cfg.Fetcher = NewRegistryFetcher(metadataRepo, metadataTag, targetsSource)
}
// create a new Updater instance
up, err := updater.New(cfg)
if err != nil {
return nil, fmt.Errorf("failed to create TUF updater instance: %w", err)
}
// try to build the top-level metadata
err = up.Refresh()
if err != nil {
return nil, fmt.Errorf("failed to refresh trusted metadata: %w", err)
}
client := &TufClient{
updater: up,
cfg: cfg,
}
return client, nil
}
// DownloadTarget downloads the target file using Updater. The Updater gets the target
// information, verifies if the target is already cached, and if it is not cached,
// downloads the target file.
func (t *TufClient) DownloadTarget(target string, filePath string) (actualFilePath string, data []byte, err error) {
// search if the desired target is available
targetInfo, err := t.updater.GetTargetInfo(target)
if err != nil {
return "", nil, err
}
// target is available, so let's see if the target is already present locally
actualFilePath, data, err = t.updater.FindCachedTarget(targetInfo, filePath)
if err != nil {
return "", nil, fmt.Errorf("failed while finding a cached target: %w", err)
}
if data != nil {
return actualFilePath, data, err
}
// target is not present locally, so let's try to download it
actualFilePath, data, err = t.updater.DownloadTarget(targetInfo, filePath, "")
if err != nil {
return "", nil, fmt.Errorf("failed to download target file %s - %w", target, err)
}
return actualFilePath, data, err
}
func (t *TufClient) GetMetadata() trustedmetadata.TrustedMetadata {
return t.updater.GetTrustedMetadataSet()
}
func (t *TufClient) MaxRootLength() int64 {
return t.cfg.RootMaxLength
}
func (t *TufClient) GetPriorRoots(metadataURL string) (map[string][]byte, error) {
rootMetadata := map[string][]byte{}
trustedMetadata := t.GetMetadata()
client := fetcher.DefaultFetcher{}
for i := 1; i < int(trustedMetadata.Root.Signed.Version); i++ {
meta, err := client.DownloadFile(metadataURL+fmt.Sprintf("/%d.root.json", i), t.MaxRootLength(), time.Second*15)
if err != nil {
return nil, fmt.Errorf("failed to download root metadata: %w", err)
}
rootMetadata[fmt.Sprintf("%d.root.json", i)] = meta
}
return rootMetadata, nil
}
func (t *TufClient) SetRemoteTargetsURL(url string) {
t.cfg.RemoteTargetsURL = url
}
// Derived from updater.loadTargets() in theupdateframework/go-tuf
func (t *TufClient) LoadDelegatedTargets(roleName, parentName string) (*metadata.Metadata[metadata.TargetsType], error) {
// extract the targets meta from the trusted snapshot metadata
meta := t.updater.GetTrustedMetadataSet()
metaInfo := meta.Snapshot.Signed.Meta[fmt.Sprintf("%s.json", roleName)]
// extract the length of the target metadata to be downloaded
length := metaInfo.Length
if length == 0 {
length = t.cfg.TargetsMaxLength
}
// extract which target metadata version should be downloaded in case of consistent snapshots
version := ""
if meta.Root.Signed.ConsistentSnapshot {
version = strconv.FormatInt(metaInfo.Version, 10)
}
// download targets metadata
data, err := t.downloadMetadata(roleName, length, version)
if err != nil {
return nil, err
}
// verify and load the new target metadata
delegatedTargets, err := meta.UpdateDelegatedTargets(data, roleName, parentName)
if err != nil {
return nil, err
}
return delegatedTargets, nil
}
// downloadMetadata download a metadata file and return it as bytes
func (t *TufClient) downloadMetadata(roleName string, length int64, version string) ([]byte, error) {
urlPath := ensureTrailingSlash(t.cfg.RemoteMetadataURL)
// build urlPath
if version == "" {
urlPath = fmt.Sprintf("%s%s.json", urlPath, url.QueryEscape(roleName))
} else {
urlPath = fmt.Sprintf("%s%s.%s.json", urlPath, version, url.QueryEscape(roleName))
}
return t.cfg.Fetcher.DownloadFile(urlPath, length, time.Second*15)
}
// ensureTrailingSlash ensures url ends with a slash
func ensureTrailingSlash(url string) string {
if updater.IsWindowsPath(url) {
slash := string(filepath.Separator)
if strings.HasSuffix(url, slash) {
return url
}
return url + slash
}
if strings.HasSuffix(url, "/") {
return url
}
return url + "/"
}

57
pkg/tuf/tuf_test.go Normal file
View File

@@ -0,0 +1,57 @@
package tuf
import (
"context"
"net/http"
"net/http/httptest"
"path/filepath"
"testing"
"github.com/docker/attest/internal/embed"
"github.com/docker/attest/internal/test"
"github.com/stretchr/testify/assert"
)
var (
HttpTufTestDataPath = filepath.Join("..", "..", "internal", "test", "testdata", "test-repo")
OciTufTestDataPath = filepath.Join("..", "..", "internal", "test", "testdata", "test-repo-oci")
)
// NewTufClient creates a new TUF client
func TestRootInit(t *testing.T) {
tufPath := test.CreateTempDir(t, "", "tuf_temp")
// Start a test HTTP server to serve data from ./testdata/test-repo/ paths
server := httptest.NewServer(http.FileServer(http.Dir(HttpTufTestDataPath)))
defer server.Close()
// run local registry
registry, regAddr := RunTestRegistry(t)
defer func() {
if err := registry.Terminate(context.Background()); err != nil {
t.Fatalf("failed to terminate container: %s", err) // nolint:gocritic
}
}()
LoadRegistryTestData(t, regAddr, OciTufTestDataPath)
testCases := []struct {
name string
metadataSource string
targetsSource string
}{
{"http", server.URL + "/metadata", server.URL + "/targets"},
{"oci", regAddr.Host + "/tuf-metadata:latest", regAddr.Host + "/tuf-targets"},
}
for _, tc := range testCases {
_, err := NewTufClient(embed.DevRoot, tufPath, tc.metadataSource, tc.targetsSource)
assert.NoErrorf(t, err, "Failed to create TUF client: %v", err)
// recreation should work with same root
_, err = NewTufClient(embed.DevRoot, tufPath, tc.metadataSource, tc.targetsSource)
assert.NoErrorf(t, err, "Failed to recreate TUF client: %v", err)
_, err = NewTufClient([]byte("broken"), tufPath, tc.metadataSource, tc.targetsSource)
assert.Errorf(t, err, "Expected error recreating TUF client with broken root: %v", err)
}
}