Rename project to actions-sync

src/git.go

@@ -0,0 +1,67 @@
+package src
+
+import (
+	"context"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
+)
+
+// A really thin Git wrapper so we can stub it out in our tests
+
+type GitImplementation interface {
+	NewGitRepository(dir string) (GitRepository, error)
+	CloneRepository(dir string, o *git.CloneOptions) (GitRepository, error)
+	RepositoryExists(dir string) bool
+}
+
+type GitRepository interface {
+	DeleteRemote(string) error
+	CreateRemote(*config.RemoteConfig) (GitRemote, error)
+	FetchContext(context.Context, *git.FetchOptions) error
+}
+
+type GitRemote interface {
+	PushContext(context.Context, *git.PushOptions) error
+	Config() *config.RemoteConfig
+}
+
+type gitImplementation struct {
+}
+
+func (i gitImplementation) NewGitRepository(dir string) (GitRepository, error) {
+	gitRepo, err := git.PlainOpen(dir)
+	if err != nil {
+		return nil, err
+	}
+	return &gitRepository{gitRepo}, nil
+}
+
+func (i gitImplementation) CloneRepository(dir string, o *git.CloneOptions) (GitRepository, error) {
+	gitRepo, err := git.PlainClone(dir, false, o)
+	if err != nil {
+		return nil, err
+	}
+	return &gitRepository{gitRepo}, nil
+}
+
+func (i gitImplementation) RepositoryExists(dir string) bool {
+	_, err := git.PlainOpen(dir)
+	return err == nil
+}
+
+type gitRepository struct {
+	inner *git.Repository
+}
+
+func (r *gitRepository) DeleteRemote(remote string) error {
+	return r.inner.DeleteRemote(remote)
+}
+
+func (r *gitRepository) CreateRemote(c *config.RemoteConfig) (GitRemote, error) {
+	return r.inner.CreateRemote(c)
+}
+
+func (r *gitRepository) FetchContext(ctx context.Context, o *git.FetchOptions) error {
+	return r.inner.FetchContext(ctx, o)
+}

src/pull.go

@@ -0,0 +1,162 @@
+package src
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+var (
+	RepoNameRegExp   = regexp.MustCompile(`^[^/]+/\S+$`)
+	ErrEmptyRepoList = errors.New("repo list cannot be empty")
+)
+
+type PullFlags struct {
+	SourceURL, RepoName, RepoNameList, RepoNameListFile string
+}
+
+func (f *PullFlags) Init(cmd *cobra.Command) {
+	cmd.Flags().StringVar(&f.SourceURL, "source-url", "https://github.com", "The domain to pull from")
+	cmd.Flags().StringVar(&f.RepoName, "repo-name", "", "Single repository name to pull")
+	cmd.Flags().StringVar(&f.RepoNameList, "repo-name-list", "", "Comma delimited list of repository names to pull")
+	cmd.Flags().StringVar(&f.RepoNameListFile, "repo-name-list-file", "", "Path to file containing a list of repository names to pull")
+}
+
+func (f *PullFlags) Validate() Validations {
+	var validations Validations
+	if !f.HasAtLeastOneRepoFlag() {
+		validations = append(validations, "one of -repo-name, -repo-name-list, -repo-name-list-file must be set")
+	}
+	return validations
+}
+
+func (f *PullFlags) HasAtLeastOneRepoFlag() bool {
+	return f.RepoName != "" || f.RepoNameList != "" || f.RepoNameListFile != ""
+}
+
+func Pull(ctx context.Context, cacheDir string, flags *PullFlags) error {
+	if flags.RepoNameList != "" {
+		repoNames, err := getRepoNamesFromCSVString(flags.RepoNameList)
+		if err != nil {
+			return err
+		}
+		return PullManyWithGitImpl(ctx, flags.SourceURL, cacheDir, repoNames, gitImplementation{})
+	}
+	if flags.RepoNameListFile != "" {
+		repoNames, err := getRepoNamesFromFile(flags.RepoNameListFile)
+		if err != nil {
+			return err
+		}
+		return PullManyWithGitImpl(ctx, flags.SourceURL, cacheDir, repoNames, gitImplementation{})
+	}
+	return PullWithGitImpl(ctx, flags.SourceURL, cacheDir, flags.RepoName, gitImplementation{})
+}
+
+func PullManyWithGitImpl(ctx context.Context, sourceURL, cacheDir string, repoNames []string, gitimpl GitImplementation) error {
+	for _, repoName := range repoNames {
+		if err := PullWithGitImpl(ctx, sourceURL, cacheDir, repoName, gitimpl); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func PullWithGitImpl(ctx context.Context, sourceURL, cacheDir string, repoName string, gitimpl GitImplementation) error {
+	repoNameParts := strings.SplitN(repoName, ":", 2)
+	originRepoName, err := validateRepoName(repoNameParts[0])
+	if err != nil {
+		return err
+	}
+	destRepoName := originRepoName
+	if len(repoNameParts) > 1 {
+		destRepoName, err = validateRepoName(repoNameParts[1])
+		if err != nil {
+			return err
+		}
+	}
+	_, err = os.Stat(cacheDir)
+	if err != nil {
+		return err
+	}
+
+	dst := path.Join(cacheDir, destRepoName)
+
+	if !gitimpl.RepositoryExists(dst) {
+		fmt.Fprintf(os.Stdout, "pulling %s to %s ...\n", originRepoName, dst)
+		_, err := gitimpl.CloneRepository(dst, &git.CloneOptions{
+			SingleBranch: false,
+			URL:          fmt.Sprintf("%s/%s", sourceURL, originRepoName),
+		})
+		if err != nil {
+			if strings.Contains(err.Error(), "authentication required") {
+				return fmt.Errorf("could not pull %s, the repository may require authentication or does not exist", originRepoName)
+			}
+			return err
+		}
+	}
+
+	repo, err := gitimpl.NewGitRepository(dst)
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(os.Stdout, "fetching * refs for %s ...\n", originRepoName)
+	err = repo.FetchContext(ctx, &git.FetchOptions{
+		RefSpecs: []config.RefSpec{
+			config.RefSpec("+refs/heads/*:refs/heads/*"),
+		},
+		Tags: git.AllTags,
+	})
+	if err != nil && err != git.NoErrAlreadyUpToDate {
+		return err
+	}
+
+	return nil
+}
+
+func getRepoNamesFromCSVString(csv string) ([]string, error) {
+	repos := filterEmptyEntries(strings.Split(csv, ","))
+	if len(repos) == 0 {
+		return nil, ErrEmptyRepoList
+	}
+	return repos, nil
+}
+
+func getRepoNamesFromFile(file string) ([]string, error) {
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	repos := filterEmptyEntries(strings.Split(string(data), "\n"))
+	if len(repos) == 0 {
+		return nil, ErrEmptyRepoList
+	}
+	return repos, nil
+}
+
+func filterEmptyEntries(names []string) []string {
+	filtered := []string{}
+	for _, name := range names {
+		if name != "" {
+			filtered = append(filtered, name)
+		}
+	}
+	return filtered
+}
+
+func validateRepoName(name string) (string, error) {
+	s := strings.TrimSpace(name)
+	if RepoNameRegExp.MatchString(s) {
+		return s, nil
+	}
+	return "", fmt.Errorf("`%s` is not a valid repo name", s)
+}

src/pull_test.go

@@ -0,0 +1 @@
+package src_test

src/push.go

@@ -0,0 +1,141 @@
+package src
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"path"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
+	"github.com/go-git/go-git/v5/plumbing/transport"
+	"github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/google/go-github/v31/github"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"golang.org/x/oauth2"
+)
+
+type PushFlags struct {
+	BaseURL, Token string
+	DisableGitAuth bool
+}
+
+func (f *PushFlags) Init(cmd *cobra.Command) {
+	cmd.Flags().StringVar(&f.BaseURL, "destination-url", "", "URL of GHES instance")
+	cmd.Flags().StringVar(&f.Token, "destination-token", "", "Token to access API on GHES instance")
+	cmd.Flags().BoolVar(&f.DisableGitAuth, "disable-push-git-auth", false, "Disables git authentication whilst pushing")
+}
+
+func (f *PushFlags) Validate() Validations {
+	var validations Validations
+	if f.BaseURL == "" {
+		validations = append(validations, "-baseURL must be set")
+	}
+	if f.Token == "" {
+		validations = append(validations, "-token must be set")
+	}
+	return validations
+}
+
+func Push(ctx context.Context, cacheDir string, flags *PushFlags) error {
+	return PushWithGitImpl(ctx, cacheDir, flags, gitImplementation{})
+}
+
+func PushWithGitImpl(ctx context.Context, cacheDir string, flags *PushFlags, gitimpl GitImplementation) error {
+	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: flags.Token})
+	tc := oauth2.NewClient(ctx, ts)
+	ghClient, err := github.NewEnterpriseClient(flags.BaseURL, flags.BaseURL, tc)
+	if err != nil {
+		return errors.Wrap(err, "error creating enterprise client")
+	}
+
+	orgDirs, err := ioutil.ReadDir(cacheDir)
+	if err != nil {
+		return errors.Wrapf(err, "error opening cache directory `%s`", cacheDir)
+	}
+	for _, orgDir := range orgDirs {
+		orgDirPath := path.Join(cacheDir, orgDir.Name())
+		if !orgDir.IsDir() {
+			return errors.Errorf("unexpected file in root of cache directory `%s`", orgDirPath)
+		}
+		repoDirs, err := ioutil.ReadDir(orgDirPath)
+		if err != nil {
+			return errors.Wrapf(err, "error opening repository cache directory `%s`", orgDirPath)
+		}
+		for _, repoDir := range repoDirs {
+			repoDirPath := path.Join(orgDirPath, repoDir.Name())
+			nwo := fmt.Sprintf("%s/%s", orgDir.Name(), repoDir.Name())
+			if !orgDir.IsDir() {
+				return errors.Errorf("unexpected file in cache directory `%s`", nwo)
+			}
+			fmt.Printf("syncing `%s`\n", nwo)
+			ghRepo, err := getOrCreateGitHubRepo(ctx, ghClient, repoDir.Name(), orgDir.Name())
+			if err != nil {
+				return errors.Wrapf(err, "error creating github repository `%s`", nwo)
+			}
+			err = syncWithCachedRepository(ctx, cacheDir, flags, ghRepo, repoDirPath, gitimpl)
+			if err != nil {
+				return errors.Wrapf(err, "error syncing repository `%s`", nwo)
+			}
+			fmt.Printf("successfully synced `%s`\n", nwo)
+		}
+	}
+	return nil
+}
+
+func getOrCreateGitHubRepo(ctx context.Context, client *github.Client, repoName, orgName string) (*github.Repository, error) {
+	repo := &github.Repository{
+		Name:        github.String(repoName),
+		HasIssues:   github.Bool(false),
+		HasWiki:     github.Bool(false),
+		HasPages:    github.Bool(false),
+		HasProjects: github.Bool(false),
+	}
+	ghRepo, resp, err := client.Repositories.Create(ctx, orgName, repo)
+	if resp.StatusCode == 422 {
+		ghRepo, _, err = client.Repositories.Get(ctx, orgName, repoName)
+	}
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating repository")
+	}
+	if ghRepo == nil {
+		return nil, errors.New("error repository is nil")
+	}
+	return ghRepo, nil
+}
+
+func syncWithCachedRepository(ctx context.Context, cacheDir string, flags *PushFlags, ghRepo *github.Repository, repoDir string, gitimpl GitImplementation) error {
+	gitRepo, err := gitimpl.NewGitRepository(repoDir)
+	if err != nil {
+		return errors.Wrapf(err, "error opening git repository %s", cacheDir)
+	}
+	_ = gitRepo.DeleteRemote("ghes")
+	remote, err := gitRepo.CreateRemote(&config.RemoteConfig{
+		Name: "ghes",
+		URLs: []string{ghRepo.GetCloneURL()},
+	})
+	if err != nil {
+		return errors.Wrap(err, "error creating remote")
+	}
+
+	var auth transport.AuthMethod
+	if !flags.DisableGitAuth {
+		auth = &http.BasicAuth{
+			Username: "username",
+			Password: flags.Token,
+		}
+	}
+	err = remote.PushContext(ctx, &git.PushOptions{
+		RemoteName: remote.Config().Name,
+		RefSpecs: []config.RefSpec{
+			"+refs/heads/*:refs/heads/*",
+			"+refs/tags/*:refs/tags/*",
+		},
+		Auth: auth,
+	})
+	if errors.Cause(err) == git.NoErrAlreadyUpToDate {
+		return nil
+	}
+	return errors.Wrapf(err, "failed to push to repo: %s", ghRepo.GetCloneURL())
+}

src/sync.go

@@ -0,0 +1,31 @@
+package src
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+)
+
+type SyncFlags struct {
+	PullFlags
+	PushFlags
+}
+
+func (f *SyncFlags) Init(cmd *cobra.Command) {
+	f.PullFlags.Init(cmd)
+	f.PushFlags.Init(cmd)
+}
+
+func (f *SyncFlags) Validate() Validations {
+	return f.PullFlags.Validate().Join(f.PushFlags.Validate())
+}
+
+func Sync(ctx context.Context, cacheDir string, flags *SyncFlags) error {
+	if err := Pull(ctx, cacheDir, &flags.PullFlags); err != nil {
+		return err
+	}
+	if err := Push(ctx, cacheDir, &flags.PushFlags); err != nil {
+		return err
+	}
+	return nil
+}

src/validation.go

@@ -0,0 +1,20 @@
+package src
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+type Validations []string
+
+func (v Validations) Error() error {
+	if len(v) <= 0 {
+		return nil
+	}
+	return errors.New("Error (Invalid Flags):\n  " + strings.Join(v, "\n  ") + "\n")
+}
+
+func (v Validations) Join(o Validations) Validations {
+	return append(v, o...)
+}