actions-sync/src/push.go

package src

import (
	"context"
	"fmt"
	"os"
	"path"
	"strings"

	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/config"
	"github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/plumbing/transport"
	"github.com/go-git/go-git/v5/plumbing/transport/http"
	"github.com/google/go-github/v43/github"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"golang.org/x/oauth2"
)

const enterpriseAegisVersionHeaderValue = "GitHub AE"
const enterpriseAPIPath = "/api/v3"
const enterpriseVersionHeaderKey = "X-GitHub-Enterprise-Version"
const xOAuthScopesHeader = "X-OAuth-Scopes"

// DefaultBatchSize of 0 means no batching (push all refs at once, original behavior)
const DefaultBatchSize = 0

// MinBatchSize is the minimum allowed batch size when batching is enabled
const MinBatchSize = 10

type PushOnlyFlags struct {
	BaseURL, Token, ActionsAdminUser string
	DisableGitAuth                   bool
	BatchSize                        int
}

type PushFlags struct {
	CommonFlags
	PushOnlyFlags
}

func (f *PushFlags) Init(cmd *cobra.Command) {
	f.CommonFlags.Init(cmd)
	f.PushOnlyFlags.Init(cmd)
}

func (f *PushOnlyFlags) Init(cmd *cobra.Command) {
	cmd.Flags().StringVar(&f.BaseURL, "destination-url", "", "URL of GHES instance")
	cmd.Flags().StringVar(&f.ActionsAdminUser, "actions-admin-user", "", "A user to impersonate for the push requests. To use the default name, pass 'actions-admin'. Note that the site_admin scope in the token is required for the impersonation to work.")
	cmd.Flags().StringVar(&f.Token, "destination-token", "", "Token to access API on GHES instance")
	cmd.Flags().BoolVar(&f.DisableGitAuth, "disable-push-git-auth", false, "Disables git authentication whilst pushing")
	cmd.Flags().IntVar(&f.BatchSize, "batch-size", DefaultBatchSize, "Number of refs to push in each batch (0 = no batching). Use a value like 100 if pushing fails for large repositories.")
}

func (f *PushFlags) Validate() Validations {
	return f.CommonFlags.Validate(false).Join(f.PushOnlyFlags.Validate())
}

func (f *PushOnlyFlags) Validate() Validations {
	var validations Validations
	if f.BaseURL == "" {
		validations = append(validations, "--destination-url must be set")
	}
	if f.Token == "" {
		validations = append(validations, "--destination-token must be set")
	}
	if f.BatchSize != 0 && f.BatchSize < MinBatchSize {
		validations = append(validations, fmt.Sprintf("--batch-size must be 0 (no batching) or at least %d", MinBatchSize))
	}
	return validations
}

func GetImpersonationToken(ctx context.Context, flags *PushFlags) (string, error) {
	fmt.Printf("getting an impersonation token for `%s` ...\n", flags.ActionsAdminUser)

	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: flags.Token})
	tc := oauth2.NewClient(ctx, ts)
	ghClient, err := github.NewEnterpriseClient(flags.BaseURL, flags.BaseURL, tc)
	if err != nil {
		return "", errors.Wrap(err, "error creating enterprise client")
	}

	rootRequest, err := ghClient.NewRequest("GET", enterpriseAPIPath, nil)
	if err != nil {
		return "", errors.Wrap(err, "error constructing request for GitHub Enterprise client.")
	}
	rootResponse, err := ghClient.Do(ctx, rootRequest, nil)
	if err != nil {
		return "", errors.Wrap(err, "error checking connectivity for GitHub Enterprise client.")
	}

	scopesHeader := rootResponse.Header.Get(xOAuthScopesHeader)
	fmt.Printf("these are the scopes we have for the current token `%s` ...\n", scopesHeader)

	if !strings.Contains(scopesHeader, "site_admin") {
		return "", errors.New("the current token doesn't have the `site_admin` scope, the impersonation function requires the `site_admin` permission to be able to impersonate")
	}

	isAE := rootResponse.Header.Get(enterpriseVersionHeaderKey) == enterpriseAegisVersionHeaderValue
	minimumRepositoryScope := "public_repo"
	if isAE {
		// the default repository scope for non-ae instances is 'public_repo'
		// while it is `repo` for ae.
		minimumRepositoryScope = "repo"
		fmt.Printf("running against GitHub AE, changing the repository scope to '%s' ...\n", minimumRepositoryScope)
	}

	impersonationToken, _, err := ghClient.Admin.CreateUserImpersonation(ctx, flags.ActionsAdminUser, &github.ImpersonateUserOptions{Scopes: []string{minimumRepositoryScope, "workflow"}})
	if err != nil {
		return "", errors.Wrap(err, "failed to impersonate Actions admin user.")
	}

	fmt.Printf("got the impersonation token for `%s` ...\n", flags.ActionsAdminUser)

	return impersonationToken.GetToken(), nil
}

func Push(ctx context.Context, flags *PushFlags) error {
	if flags.ActionsAdminUser != "" {
		var token, err = GetImpersonationToken(ctx, flags)
		if err != nil {
			return errors.Wrap(err, "error obtaining the impersonation token")
		}

		// Override the initial token with the one that we got in the exchange
		flags.Token = token
	} else {
		fmt.Print("not using impersonation for the requests \n")
	}

	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: flags.Token})
	tc := oauth2.NewClient(ctx, ts)
	ghClient, err := github.NewEnterpriseClient(flags.BaseURL, flags.BaseURL, tc)
	if err != nil {
		return errors.Wrap(err, "error creating enterprise client")
	}

	repoNames, err := getRepoNamesFromRepoFlags(&flags.CommonFlags)
	if err != nil {
		return err
	}

	if repoNames == nil {
		repoNames, err = getRepoNamesFromCacheDir(&flags.CommonFlags)
		if err != nil {
			return err
		}
	}

	return PushManyWithGitImpl(ctx, flags, repoNames, ghClient, gitImplementation{})
}

func PushManyWithGitImpl(ctx context.Context, flags *PushFlags, repoNames []string, ghClient *github.Client, gitimpl GitImplementation) error {
	for _, repoName := range repoNames {
		if err := PushWithGitImpl(ctx, flags, repoName, ghClient, gitimpl); err != nil {
			return err
		}
	}
	return nil
}

func PushWithGitImpl(ctx context.Context, flags *PushFlags, repoName string, ghClient *github.Client, gitimpl GitImplementation) error {
	_, nwo, err := extractSourceDest(repoName)
	if err != nil {
		return err
	}

	ownerName, bareRepoName, err := splitNwo(nwo)
	if err != nil {
		return err
	}

	repoDirPath := path.Join(flags.CacheDir, nwo)
	_, err = os.Stat(repoDirPath)
	if err != nil {
		return err
	}

	fmt.Printf("syncing `%s`\n", nwo)
	ghRepo, err := getOrCreateGitHubRepo(ctx, ghClient, bareRepoName, ownerName)
	if err != nil {
		return errors.Wrapf(err, "error creating github repository `%s`", nwo)
	}
	err = syncWithCachedRepository(ctx, flags, ghRepo, repoDirPath, gitimpl)
	if err != nil {
		return errors.Wrapf(err, "error syncing repository `%s`", nwo)
	}
	fmt.Printf("successfully synced `%s`\n", nwo)
	return nil
}

func getOrCreateGitHubRepo(ctx context.Context, client *github.Client, repoName, ownerName string) (*github.Repository, error) {
	// retrieve user associated to authentication credentials provided
	currentUser, userResponse, err := client.Users.Get(ctx, "")
	if err != nil {
		return nil, errors.Wrap(err, "error retrieving authenticated user")
	}
	if currentUser == nil || currentUser.Login == nil {
		return nil, errors.New("error retrieving authenticated user's login name")
	}
	// checking if we talk to GHAE
	isAE := userResponse.Header.Get(enterpriseVersionHeaderKey) == enterpriseAegisVersionHeaderValue

	// check if the owner refers to the authenticated user or an organization.
	var createRepoOrgName string
	if strings.EqualFold(*currentUser.Login, ownerName) {
		// we'll create the repo under the authenticated user's account.
		createRepoOrgName = ""
	} else {
		// ensure the org exists.
		createRepoOrgName = ownerName
		_, err := getOrCreateGitHubOrg(ctx, client, ownerName, *currentUser.Login)
		if err != nil {
			return nil, err
		}
	}

	// check if repository already exists
	ghRepo, resp, err := client.Repositories.Get(ctx, ownerName, repoName)

	if err == nil {
		fmt.Printf("Existing repo `%s/%s`\n", ownerName, repoName)
	} else if resp != nil && resp.StatusCode == 404 {
		// repo not existing yet - try to create
		visibility := github.String("public")
		if isAE {
			visibility = github.String("internal")
		}
		repo := &github.Repository{
			Name:        github.String(repoName),
			HasIssues:   github.Bool(false),
			HasWiki:     github.Bool(false),
			HasPages:    github.Bool(false),
			HasProjects: github.Bool(false),
			Visibility:  visibility,
		}

		ghRepo, _, err = client.Repositories.Create(ctx, createRepoOrgName, repo)
		if err == nil {
			fmt.Printf("Created repo `%s/%s`\n", ownerName, repoName)
		} else {
			return nil, errors.Wrapf(err, "error creating repository %s/%s", ownerName, repoName)
		}
	} else if err != nil {
		return nil, errors.Wrapf(err, "error creating repository %s/%s", ownerName, repoName)
	}

	if ghRepo == nil {
		return nil, errors.New("error repository is nil")
	}
	return ghRepo, nil
}

func getOrCreateGitHubOrg(ctx context.Context, client *github.Client, orgName, admin string) (*github.Organization, error) {
	org := &github.Organization{Login: &orgName}

	var getErr error
	ghOrg, _, createErr := client.Admin.CreateOrg(ctx, org, admin)
	if createErr == nil {
		fmt.Printf("Created organization `%s` (admin: %s)\n", orgName, admin)
	} else {
		// Regardless of why create failed, see if we can retrieve the org
		ghOrg, _, getErr = client.Organizations.Get(ctx, orgName)
	}
	if createErr != nil && getErr != nil {
		return nil, errors.Wrapf(createErr, "error creating organization %s", orgName)
	}
	if ghOrg == nil {
		return nil, errors.New("error organization is nil")
	}

	return ghOrg, nil
}

func syncWithCachedRepository(ctx context.Context, flags *PushFlags, ghRepo *github.Repository, repoDir string, gitimpl GitImplementation) error {
	gitRepo, err := gitimpl.NewGitRepository(repoDir)
	if err != nil {
		return errors.Wrapf(err, "error opening git repository %s", repoDir)
	}
	_ = gitRepo.DeleteRemote("ghes")
	remote, err := gitRepo.CreateRemote(&config.RemoteConfig{
		Name: "ghes",
		URLs: []string{ghRepo.GetCloneURL()},
	})
	if err != nil {
		return errors.Wrap(err, "error creating remote")
	}

	var auth transport.AuthMethod
	if !flags.DisableGitAuth {
		auth = &http.BasicAuth{
			Username: "username",
			Password: flags.Token,
		}
	}

	// If batch size is 0 or negative, use original wildcard approach (no batching)
	if flags.BatchSize <= 0 {
		err = remote.PushContext(ctx, &git.PushOptions{
			RemoteName: remote.Config().Name,
			RefSpecs: []config.RefSpec{
				"+refs/heads/*:refs/heads/*",
				"+refs/tags/*:refs/tags/*",
			},
			Auth: auth,
		})
		if errors.Cause(err) == git.NoErrAlreadyUpToDate {
			return nil
		}
		return errors.Wrapf(err, "failed to push to repo: %s", ghRepo.GetCloneURL())
	}

	// Batching requested - collect all refs and push in batches
	refs, err := collectRefs(gitRepo)
	if err != nil {
		return errors.Wrap(err, "error collecting refs")
	}

	return pushRefsInBatches(ctx, remote, refs, flags.BatchSize, auth, ghRepo.GetCloneURL())
}

// collectRefs gathers all branch and tag refs from the repository
func collectRefs(gitRepo GitRepository) ([]plumbing.ReferenceName, error) {
	refIter, err := gitRepo.References()
	if err != nil {
		return nil, err
	}

	var refs []plumbing.ReferenceName
	err = refIter.ForEach(func(ref *plumbing.Reference) error {
		name := ref.Name()
		// Only include branches and tags
		if name.IsBranch() || name.IsTag() {
			refs = append(refs, name)
		}
		return nil
	})
	if err != nil {
		return nil, err
	}

	return refs, nil
}

// pushRefsInBatches pushes refs in smaller batches to avoid server-side limits
func pushRefsInBatches(ctx context.Context, remote GitRemote, refs []plumbing.ReferenceName, batchSize int, auth transport.AuthMethod, cloneURL string) error {
	totalRefs := len(refs)

	for i := 0; i < totalRefs; i += batchSize {
		end := i + batchSize
		if end > totalRefs {
			end = totalRefs
		}

		batch := refs[i:end]
		refSpecs := make([]config.RefSpec, len(batch))
		for j, ref := range batch {
			// Create a refspec like "+refs/heads/main:refs/heads/main"
			refSpecs[j] = config.RefSpec("+" + ref.String() + ":" + ref.String())
		}

		err := remote.PushContext(ctx, &git.PushOptions{
			RemoteName: remote.Config().Name,
			RefSpecs:   refSpecs,
			Auth:       auth,
		})

		if err != nil {
			if errors.Cause(err) == git.NoErrAlreadyUpToDate {
				// This batch was already up to date, continue to next batch
				continue
			}
			return errors.Wrapf(err, "failed to push batch %d-%d of %d refs to repo: %s", i+1, end, totalRefs, cloneURL)
		}
	}

	return nil
}