diff options
| author | jannfis <jann@mistrust.net> | 2021-10-26 22:43:34 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-10-26 22:43:34 +0200 |
| commit | a03f31915d71ae39f1c9ac5d7d4d1501d11d158b (patch) | |
| tree | c9a87b7da7d4505e18b7bdc2344ef011e8fc037c /ext | |
| parent | b4f28e8f48d38559b23453e1e913f6bb3361fc25 (diff) | |
chore: Update and refactor Git client (#283)
Signed-off-by: jannfis <jann@mistrust.net>
Diffstat (limited to 'ext')
| -rw-r--r-- | ext/git/client.go | 270 | ||||
| -rw-r--r-- | ext/git/creds.go | 198 | ||||
| -rw-r--r-- | ext/git/git.go | 10 | ||||
| -rw-r--r-- | ext/git/git_test.go | 50 | ||||
| -rw-r--r-- | ext/git/mocks/Client.go | 24 | ||||
| -rw-r--r-- | ext/git/mocks/Creds.go | 2 | ||||
| -rw-r--r-- | ext/git/mocks/GenericHTTPSCreds.go | 88 | ||||
| -rw-r--r-- | ext/git/mocks/gitRefCache.go | 41 | ||||
| -rw-r--r-- | ext/git/ssh.go | 59 | ||||
| -rw-r--r-- | ext/git/workaround.go | 36 | ||||
| -rw-r--r-- | ext/git/writer.go | 118 |
11 files changed, 673 insertions, 223 deletions
diff --git a/ext/git/client.go b/ext/git/client.go index 92d8434..fa470bc 100644 --- a/ext/git/client.go +++ b/ext/git/client.go @@ -9,26 +9,26 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "sort" "strconv" "strings" "time" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport" + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/storage/memory" + log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" - "gopkg.in/src-d/go-git.v4" - "gopkg.in/src-d/go-git.v4/config" - "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/transport" - githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" - ssh2 "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh" - "gopkg.in/src-d/go-git.v4/storage/memory" "github.com/argoproj/argo-cd/v2/common" certutil "github.com/argoproj/argo-cd/v2/util/cert" executil "github.com/argoproj/argo-cd/v2/util/exec" - - "github.com/argoproj-labs/argocd-image-updater/pkg/log" + "github.com/argoproj/argo-cd/v2/util/proxy" ) type RevisionMetadata struct { @@ -45,24 +45,16 @@ type Refs struct { // heads and remotes are also refs, but are not needed at this time. } -// CommitOptions holds options for a git commit operation -type CommitOptions struct { - // CommitMessageText holds a short commit message (-m option) - CommitMessageText string - // CommitMessagePath holds the path to a file to be used for the commit message (-F option) - CommitMessagePath string - // SigningKey holds a GnuPG key ID used to sign the commit with (-S option) - SigningKey string - // SignOff specifies whether to sign-off a commit (-s option) - SignOff bool +type gitRefCache interface { + SetGitReferences(repo string, references []*plumbing.Reference) error + GetGitReferences(repo string, references *[]*plumbing.Reference) error } // Client is a generic git client interface type Client interface { Root() string Init() error - Fetch() error - FetchRef(ref string) error + Fetch(revision string) error Checkout(revision string) error LsRefs() (*Refs, error) LsRemote(revision string) (string, error) @@ -71,16 +63,23 @@ type Client interface { CommitSHA() (string, error) RevisionMetadata(revision string) (*RevisionMetadata, error) VerifyCommitSignature(string) (string, error) - Branch(sourceBranch, targetBranch string) error Commit(pathSpec string, opts *CommitOptions) error + Branch(sourceBranch string, targetBranch string) error Push(remote string, branch string, force bool) error Add(path string) error SymRefToBranch(symRef string) (string, error) Config(username string, email string) error } +type EventHandlers struct { + OnLsRemote func(repo string) func() + OnFetch func(repo string) func() +} + // nativeGitClient implements Client interface using git CLI type nativeGitClient struct { + EventHandlers + // URL of the repository repoURL string // Root path of repository @@ -91,6 +90,12 @@ type nativeGitClient struct { insecure bool // Whether the repository is LFS enabled enableLfs bool + // gitRefCache knows how to cache git refs + gitRefCache gitRefCache + // indicates if client allowed to load refs from cache + loadRefFromCache bool + // HTTP/HTTPS proxy used to access repository + proxy string } var ( @@ -107,23 +112,45 @@ func init() { } } -func NewClient(rawRepoURL string, creds Creds, insecure bool, enableLfs bool) (Client, error) { - root := filepath.Join(os.TempDir(), strings.Replace(NormalizeGitURL(rawRepoURL), "/", "_", -1)) +type ClientOpts func(c *nativeGitClient) + +// WithCache sets git revisions cacher as well as specifies if client should tries to use cached resolved revision +func WithCache(cache gitRefCache, loadRefFromCache bool) ClientOpts { + return func(c *nativeGitClient) { + c.gitRefCache = cache + c.loadRefFromCache = loadRefFromCache + } +} + +// WithEventHandlers sets the git client event handlers +func WithEventHandlers(handlers EventHandlers) ClientOpts { + return func(c *nativeGitClient) { + c.EventHandlers = handlers + } +} + +func NewClient(rawRepoURL string, creds Creds, insecure bool, enableLfs bool, proxy string, opts ...ClientOpts) (Client, error) { + r := regexp.MustCompile("(/|:)") + root := filepath.Join(os.TempDir(), r.ReplaceAllString(NormalizeGitURL(rawRepoURL), "_")) if root == os.TempDir() { return nil, fmt.Errorf("Repository '%s' cannot be initialized, because its root would be system temp at %s", rawRepoURL, root) } - return NewClientExt(rawRepoURL, root, creds, insecure, enableLfs) + return NewClientExt(rawRepoURL, root, creds, insecure, enableLfs, proxy, opts...) } -func NewClientExt(rawRepoURL string, root string, creds Creds, insecure bool, enableLfs bool) (Client, error) { - client := nativeGitClient{ +func NewClientExt(rawRepoURL string, root string, creds Creds, insecure bool, enableLfs bool, proxy string, opts ...ClientOpts) (Client, error) { + client := &nativeGitClient{ repoURL: rawRepoURL, root: root, creds: creds, insecure: insecure, enableLfs: enableLfs, + proxy: proxy, } - return &client, nil + for i := range opts { + opts[i](client) + } + return client, nil } // Returns a HTTP client object suitable for go-git to use using the following @@ -134,7 +161,7 @@ func NewClientExt(rawRepoURL string, root string, creds Creds, insecure bool, en // a client with those certificates in the list of root CAs used to verify // the server's certificate. // - Otherwise (and on non-fatal errors), a default HTTP client is returned. -func GetRepoHTTPClient(repoURL string, insecure bool, creds Creds) *http.Client { +func GetRepoHTTPClient(repoURL string, insecure bool, creds Creds, proxyURL string) *http.Client { // Default HTTP client var customHTTPClient = &http.Client{ // 15 second timeout @@ -145,22 +172,24 @@ func GetRepoHTTPClient(repoURL string, insecure bool, creds Creds) *http.Client }, } + proxyFunc := proxy.GetCallback(proxyURL) + // Callback function to return any configured client certificate // We never return err, but an empty cert instead. clientCertFunc := func(req *tls.CertificateRequestInfo) (*tls.Certificate, error) { var err error cert := tls.Certificate{} - // If we aren't called with HTTPSCreds, then we just return an empty cert - httpsCreds, ok := creds.(HTTPSCreds) + // If we aren't called with GenericHTTPSCreds, then we just return an empty cert + httpsCreds, ok := creds.(GenericHTTPSCreds) if !ok { return &cert, nil } // If the creds contain client certificate data, we return a TLS.Certificate // populated with the cert and its key. - if httpsCreds.clientCertData != "" && httpsCreds.clientCertKey != "" { - cert, err = tls.X509KeyPair([]byte(httpsCreds.clientCertData), []byte(httpsCreds.clientCertKey)) + if httpsCreds.HasClientCert() { + cert, err = tls.X509KeyPair([]byte(httpsCreds.GetClientCertData()), []byte(httpsCreds.GetClientCertKey())) if err != nil { log.Errorf("Could not load Client Certificate: %v", err) return &cert, nil @@ -172,7 +201,7 @@ func GetRepoHTTPClient(repoURL string, insecure bool, creds Creds) *http.Client if insecure { customHTTPClient.Transport = &http.Transport{ - Proxy: http.ProxyFromEnvironment, + Proxy: proxyFunc, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, GetClientCertificate: clientCertFunc, @@ -190,7 +219,7 @@ func GetRepoHTTPClient(repoURL string, insecure bool, creds Creds) *http.Client } else if len(serverCertificatePem) > 0 { certPool := certutil.GetCertPoolFromPEMData(serverCertificatePem) customHTTPClient.Transport = &http.Transport{ - Proxy: http.ProxyFromEnvironment, + Proxy: proxyFunc, TLSClientConfig: &tls.Config{ RootCAs: certPool, GetClientCertificate: clientCertFunc, @@ -200,7 +229,7 @@ func GetRepoHTTPClient(repoURL string, insecure bool, creds Creds) *http.Client } else { // else no custom certificate stored. customHTTPClient.Transport = &http.Transport{ - Proxy: http.ProxyFromEnvironment, + Proxy: proxyFunc, TLSClientConfig: &tls.Config{ GetClientCertificate: clientCertFunc, }, @@ -223,7 +252,9 @@ func newAuth(repoURL string, creds Creds) (transport.AuthMethod, error) { if err != nil { return nil, err } - auth := &ssh2.PublicKeys{User: sshUser, Signer: signer} + auth := &PublicKeysWithOptions{} + auth.User = sshUser + auth.Signer = signer if creds.insecure { auth.HostKeyCallback = ssh.InsecureIgnoreHostKey() } else { @@ -238,6 +269,13 @@ func newAuth(repoURL string, creds Creds) (transport.AuthMethod, error) { case HTTPSCreds: auth := githttp.BasicAuth{Username: creds.username, Password: creds.password} return &auth, nil + case GitHubAppCreds: + token, err := creds.getAccessToken() + if err != nil { + return nil, err + } + auth := githttp.BasicAuth{Username: "x-access-token", Password: token} + return &auth, nil } return nil, nil } @@ -281,24 +319,18 @@ func (m *nativeGitClient) IsLFSEnabled() bool { } // Fetch fetches latest updates from origin -func (m *nativeGitClient) Fetch() error { - err := m.runCredentialedCmd("git", "fetch", "origin", "--tags", "--force") - // When we have LFS support enabled, check for large files and fetch them too. - if err == nil && m.IsLFSEnabled() { - largeFiles, err := m.LsLargeFiles() - if err == nil && len(largeFiles) > 0 { - err = m.runCredentialedCmd("git", "lfs", "fetch", "--all") - if err != nil { - return err - } - } +func (m *nativeGitClient) Fetch(revision string) error { + if m.OnFetch != nil { + done := m.OnFetch(m.repoURL) + defer done() } - return err -} -// Fetch fetches latest updates from origin -func (m *nativeGitClient) FetchRef(ref string) error { - err := m.runCredentialedCmd("git", "fetch", "--tags", "--force") + var err error + if revision != "" { + err = m.runCredentialedCmd("git", "fetch", "origin", revision, "--tags", "--force") + } else { + err = m.runCredentialedCmd("git", "fetch", "origin", "--tags", "--force") + } // When we have LFS support enabled, check for large files and fetch them too. if err == nil && m.IsLFSEnabled() { largeFiles, err := m.LsLargeFiles() @@ -333,7 +365,7 @@ func (m *nativeGitClient) LsLargeFiles() ([]string, error) { return ss, nil } -// Checkout checkout specified git sha +// Checkout checkout specified revision func (m *nativeGitClient) Checkout(revision string) error { if revision == "" || revision == "HEAD" { revision = "origin/HEAD" @@ -368,6 +400,18 @@ func (m *nativeGitClient) Checkout(revision string) error { } func (m *nativeGitClient) getRefs() ([]*plumbing.Reference, error) { + if m.gitRefCache != nil && m.loadRefFromCache { + var res []*plumbing.Reference + if m.gitRefCache.GetGitReferences(m.repoURL, &res) == nil { + return res, nil + } + } + + if m.OnLsRemote != nil { + done := m.OnLsRemote(m.repoURL) + defer done() + } + repo, err := git.Init(memory.NewStorage(), nil) if err != nil { return nil, err @@ -383,7 +427,14 @@ func (m *nativeGitClient) getRefs() ([]*plumbing.Reference, error) { if err != nil { return nil, err } - return listRemote(remote, &git.ListOptions{Auth: auth}, m.insecure, m.creds) + res, err := listRemote(remote, &git.ListOptions{Auth: auth}, m.insecure, m.creds, m.proxy) + if err == nil && m.gitRefCache != nil { + if err := m.gitRefCache.SetGitReferences(m.repoURL, res); err != nil { + log.Warnf("Failed to store git references to cache: %v", err) + } + return res, nil + } + return res, err } func (m *nativeGitClient) LsRefs() (*Refs, error) { @@ -450,16 +501,12 @@ func (m *nativeGitClient) lsRemote(revision string) (string, error) { refToResolve := "" for _, ref := range refs { refName := ref.Name().String() - if refName != "HEAD" && !strings.HasPrefix(refName, "refs/heads/") && !strings.HasPrefix(refName, "refs/tags/") { - // ignore things like 'refs/pull/' 'refs/reviewable' - continue - } hash := ref.Hash().String() if ref.Type() == plumbing.HashReference { refToHash[refName] = hash } //log.Debugf("%s\t%s", hash, refName) - if ref.Name().Short() == revision { + if ref.Name().Short() == revision || refName == revision { if ref.Type() == plumbing.HashReference { log.Debugf("revision '%s' resolved to '%s'", revision, hash) return hash, nil @@ -528,108 +575,10 @@ func (m *nativeGitClient) VerifyCommitSignature(revision string) (string, error) return out, nil } -// Commit perfoms a git commit for the given pathSpec to the currently checked -// out branch. If pathSpec is empty, or the special value "*", all pending -// changes will be commited. If message is not the empty string, it will be -// used as the commit message, otherwise a default commit message will be used. -// If signingKey is not the empty string, commit will be signed with the given -// GPG key. -func (m *nativeGitClient) Commit(pathSpec string, opts *CommitOptions) error { - defaultCommitMsg := "Update parameters" - args := []string{"commit"} - if pathSpec == "" || pathSpec == "*" { - args = append(args, "-a") - } - if opts.SigningKey != "" { - args = append(args, "-S", opts.SigningKey) - } - if opts.SignOff { - args = append(args, "-s") - } - if opts.CommitMessageText != "" { - args = append(args, "-m", opts.CommitMessageText) - } else if opts.CommitMessagePath != "" { - args = append(args, "-F", opts.CommitMessagePath) - } else { - args = append(args, "-m", defaultCommitMsg) - } - - out, err := m.runCmd(args...) - if err != nil { - log.Errorf(out) - return err - } - - return nil -} - -// Branch creates a new target branch from a given source branch -func (m *nativeGitClient) Branch(sourceBranch string, targetBranch string) error { - if sourceBranch != "" { - _, err := m.runCmd("checkout", sourceBranch) - if err != nil { - return fmt.Errorf("could not checkout source branch: %v", err) - } - } - - _, err := m.runCmd("branch", targetBranch) - if err != nil { - return fmt.Errorf("could not create new branch: %v", err) - } - - return nil -} - -// Push pushes local changes to the remote branch. If force is true, will force -// the remote to accept the push. -func (m *nativeGitClient) Push(remote string, branch string, force bool) error { - args := []string{"push"} - if force { - args = append(args, "-f") - } - args = append(args, remote, branch) - err := m.runCredentialedCmd("git", args...) - if err != nil { - return fmt.Errorf("could not push %s to %s: %v", branch, remote, err) - } - return nil -} - -// Add adds a path spec to the repository -func (m *nativeGitClient) Add(path string) error { - return m.runCredentialedCmd("git", "add", path) -} - -// SymRefToBranch retrieves the branch name a symbolic ref points to -func (m *nativeGitClient) SymRefToBranch(symRef string) (string, error) { - output, err := m.runCmd("symbolic-ref", symRef) - if err != nil { - return "", fmt.Errorf("could not resolve symbolic ref '%s': %v", symRef, err) - } - if a := strings.SplitN(output, "refs/heads/", 2); len(a) == 2 { - return a[1], nil - } - return "", fmt.Errorf("no symbolic ref named '%s' could be found", symRef) -} - -// Config configures username and email address for the repository -func (m *nativeGitClient) Config(username string, email string) error { - _, err := m.runCmd("config", "user.name", username) - if err != nil { - return fmt.Errorf("could not set git username: %v", err) - } - _, err = m.runCmd("config", "user.email", email) - if err != nil { - return fmt.Errorf("could not set git email: %v", err) - } - - return nil -} - // runWrapper runs a custom command with all the semantics of running the Git client func (m *nativeGitClient) runGnuPGWrapper(wrapper string, args ...string) (string, error) { cmd := exec.Command(wrapper, args...) - cmd.Env = append(cmd.Env, fmt.Sprintf("GNUPGHOME=%s", common.GetGnuPGHomePath())) + cmd.Env = append(cmd.Env, fmt.Sprintf("GNUPGHOME=%s", common.GetGnuPGHomePath()), "LANG=C") return m.runCmdOutput(cmd) } @@ -682,5 +631,8 @@ func (m *nativeGitClient) runCmdOutput(cmd *exec.Cmd) (string, error) { } } } + + cmd.Env = proxy.UpsertEnv(cmd, m.proxy) + return executil.Run(cmd) } diff --git a/ext/git/creds.go b/ext/git/creds.go index 25a98cf..d326277 100644 --- a/ext/git/creds.go +++ b/ext/git/creds.go @@ -1,18 +1,43 @@ package git import ( + "context" + "crypto/sha256" "fmt" "io" "io/ioutil" "os" + "strconv" "strings" + "time" + + gocache "github.com/patrickmn/go-cache" argoio "github.com/argoproj/gitops-engine/pkg/utils/io" + "github.com/bradleyfalzon/ghinstallation" log "github.com/sirupsen/logrus" + "github.com/argoproj/argo-cd/v2/common" + certutil "github.com/argoproj/argo-cd/v2/util/cert" ) +// In memory cache for storing github APP api token credentials +var ( + githubAppTokenCache *gocache.Cache +) + +func init() { + githubAppCredsExp := common.GithubAppCredsExpirationDuration + if exp := os.Getenv(common.EnvGithubAppCredsExpirationDuration); exp != "" { + if qps, err := strconv.Atoi(exp); err != nil { + githubAppCredsExp = time.Duration(qps) * time.Minute + } + } + + githubAppTokenCache = gocache.New(githubAppCredsExp, 1*time.Minute) +} + type Creds interface { Environ() (io.Closer, []string, error) } @@ -25,6 +50,8 @@ func (c NopCloser) Close() error { return nil } +var _ Creds = NopCreds{} + type NopCreds struct { } @@ -32,6 +59,17 @@ func (c NopCreds) Environ() (io.Closer, []string, error) { return NopCloser{}, nil, nil } +var _ io.Closer = NopCloser{} + +type GenericHTTPSCreds interface { + HasClientCert() bool + GetClientCertData() string + GetClientCertKey() string + Environ() (io.Closer, []string, error) +} + +var _ GenericHTTPSCreds = HTTPSCreds{} + // HTTPS creds implementation type HTTPSCreds struct { // Username for authentication @@ -44,15 +82,18 @@ type HTTPSCreds struct { clientCertData string // Client certificate key to use clientCertKey string + // HTTP/HTTPS proxy used to access repository + proxy string } -func NewHTTPSCreds(username string, password string, clientCertData string, clientCertKey string, insecure bool) HTTPSCreds { +func NewHTTPSCreds(username string, password string, clientCertData string, clientCertKey string, insecure bool, proxy string) GenericHTTPSCreds { return HTTPSCreds{ username, password, insecure, clientCertData, clientCertKey, + proxy, } } @@ -71,7 +112,7 @@ func (c HTTPSCreds) Environ() (io.Closer, []string, error) { // In case the repo is configured for using a TLS client cert, we need to make // sure git client will use it. The certificate's key must not be password // protected. - if c.clientCertData != "" && c.clientCertKey != "" { + if c.HasClientCert() { var certFile, keyFile *os.File // We need to actually create two temp files, one for storing cert data and @@ -116,6 +157,18 @@ func (c HTTPSCreds) Environ() (io.Closer, []string, error) { return httpCloser, env, nil } +func (g HTTPSCreds) HasClientCert() bool { + return g.clientCertData != "" && g.clientCertKey != "" +} + +func (c HTTPSCreds) GetClientCertData() string { + return c.clientCertData +} + +func (c HTTPSCreds) GetClientCertKey() string { + return c.clientCertKey +} + // SSH implementation type SSHCreds struct { sshPrivateKey string @@ -179,3 +232,144 @@ func (c SSHCreds) Environ() (io.Closer, []string, error) { env = append(env, []string{fmt.Sprintf("GIT_SSH_COMMAND=%s", strings.Join(args, " "))}...) return sshPrivateKeyFile(file.Name()), env, nil } + +// GitHubAppCreds to authenticate as GitHub application +type GitHubAppCreds struct { + appID int64 + appInstallId int64 + privateKey string + baseURL string + repoURL string + clientCertData string + clientCertKey string + insecure bool + proxy string +} + +// NewGitHubAppCreds provide github app credentials +func NewGitHubAppCreds(appID int64, appInstallId int64, privateKey string, baseURL string, repoURL string, clientCertData string, clientCertKey string, insecure bool) GenericHTTPSCreds { + return GitHubAppCreds{appID: appID, appInstallId: appInstallId, privateKey: privateKey, baseURL: baseURL, repoURL: repoURL, clientCertData: clientCertData, clientCertKey: clientCertKey, insecure: insecure} +} + +func (g GitHubAppCreds) Environ() (io.Closer, []string, error) { + token, err := g.getAccessToken() + if err != nil { + return NopCloser{}, nil, err + } + + env := []string{fmt.Sprintf("GIT_ASKPASS=%s", "git-ask-pass.sh"), "GIT_USERNAME=x-access-token", fmt.Sprintf("GIT_PASSWORD=%s", token)} + httpCloser := authFilePaths(make([]string, 0)) + + // GIT_SSL_NO_VERIFY is used to tell git not to validate the server's cert at + // all. + if g.insecure { + env = append(env, "GIT_SSL_NO_VERIFY=true") + } + + // In case the repo is configured for using a TLS client cert, we need to make + // sure git client will use it. The certificate's key must not be password + // protected. + if g.HasClientCert() { + var certFile, keyFile *os.File + + // We need to actually create two temp files, one for storing cert data and + // another for storing the key. If we fail to create second fail, the first + // must be removed. + certFile, err := ioutil.TempFile(argoio.TempDir, "") + if err == nil { + defer certFile.Close() + keyFile, err = ioutil.TempFile(argoio.TempDir, "") + if err != nil { + removeErr := os.Remove(certFile.Name()) + if removeErr != nil { + log.Errorf("Could not remove previously created tempfile %s: %v", certFile.Name(), removeErr) + } + return NopCloser{}, nil, err + } + defer keyFile.Close() + } else { + return NopCloser{}, nil, err + } + + // We should have both temp files by now + httpCloser = authFilePaths([]string{certFile.Name(), keyFile.Name()}) + + _, err = certFile.WriteString(g.clientCertData) + if err != nil { + httpCloser.Close() + return NopCloser{}, nil, err + } + // GIT_SSL_CERT is the full path to a client certificate to be used + env = append(env, fmt.Sprintf("GIT_SSL_CERT=%s", certFile.Name())) + + _, err = keyFile.WriteString(g.clientCertKey) + if err != nil { + httpCloser.Close() + return NopCloser{}, nil, err + } + // GIT_SSL_KEY is the full path to a client certificate's key to be used + env = append(env, fmt.Sprintf("GIT_SSL_KEY=%s", keyFile.Name())) + + } + return httpCloser, env, nil +} + +// getAccessToken fetches GitHub token using the app id, install id, and private key. +// the token is then cached for re-use. +func (g GitHubAppCreds) getAccessToken() (string, error) { + // Timeout + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + // Compute hash of creds for lookup in cache + h := sha256.New() + _, err := h.Write([]byte(fmt.Sprintf("%s %d %d %s", g.privateKey, g.appID, g.appInstallId, g.baseURL))) + if err != nil { + return "", err + } + key := fmt.Sprintf("%x", h.Sum(nil)) + + // Check cache for GitHub transport which helps fetch an API token + t, found := githubAppTokenCache.Get(key) + if found { + itr := t.(*ghinstallation.Transport) + // This method caches the token and if it's expired retrieves a new one + return itr.Token(ctx) + } + + // GitHub API url + baseUrl := "https://api.github.com" + if g.baseURL != "" { + baseUrl = strings.TrimSuffix(g.baseURL, "/") + } + + // Create a new GitHub transport + c := GetRepoHTTPClient(baseUrl, g.insecure, g, g.proxy) + itr, err := ghinstallation.New(c.Transport, + g.appID, + g.appInstallId, + []byte(g.privateKey), + ) + if err != nil { + return "", err + } + + itr.BaseURL = baseUrl + + // Add transport to cache + githubAppTokenCache.Set(key, itr, time.Minute*60) + + return itr.Token(ctx) +} + +func (g GitHubAppCreds) HasClientCert() bool { + return g.clientCertData != "" && g.clientCertKey != "" +} + +func (g GitHubAppCreds) GetClientCertData() string { + return g.clientCertData +} + +func (g GitHubAppCreds) GetClientCertKey() string { + return g.clientCertKey +} diff --git a/ext/git/git.go b/ext/git/git.go index 64d3587..b925789 100644 --- a/ext/git/git.go +++ b/ext/git/git.go @@ -26,6 +26,7 @@ var ( commitSHARegex = regexp.MustCompile("^[0-9A-Fa-f]{40}$") sshURLRegex = regexp.MustCompile("^(ssh://)?([^/:]*?)@[^@]+$") httpsURLRegex = regexp.MustCompile("^(https://).*") + httpURLRegex = regexp.MustCompile("^(http://).*") ) // IsCommitSHA returns whether or not a string is a 40 character SHA-1 @@ -84,9 +85,14 @@ func IsHTTPSURL(url string) bool { return httpsURLRegex.MatchString(url) } +// IsHTTPURL returns true if supplied URL is HTTP URL +func IsHTTPURL(url string) bool { + return httpURLRegex.MatchString(url) +} + // TestRepo tests if a repo exists and is accessible with the given credentials -func TestRepo(repo string, creds Creds, insecure bool, enableLfs bool) error { - clnt, err := NewClient(repo, creds, insecure, enableLfs) +func TestRepo(repo string, creds Creds, insecure bool, enableLfs bool, proxy string) error { + clnt, err := NewClient(repo, creds, insecure, enableLfs, proxy) if err != nil { return err } diff --git a/ext/git/git_test.go b/ext/git/git_test.go index 68bdbd2..8738b75 100644 --- a/ext/git/git_test.go +++ b/ext/git/git_test.go @@ -129,11 +129,11 @@ func TestSameURL(t *testing.T) { } func TestCustomHTTPClient(t *testing.T) { - certFile, err := filepath.Abs("testdata/certs/argocd-test-client.crt") + certFile, err := filepath.Abs("../../test/fixture/certs/argocd-test-client.crt") assert.NoError(t, err) assert.NotEqual(t, "", certFile) - keyFile, err := filepath.Abs("testdata/certs/argocd-test-client.key") + keyFile, err := filepath.Abs("../../test/fixture/certs/argocd-test-client.key") assert.NoError(t, err) assert.NotEqual(t, "", keyFile) @@ -146,8 +146,8 @@ func TestCustomHTTPClient(t *testing.T) { assert.NotEqual(t, "", string(keyData)) // Get HTTPSCreds with client cert creds specified, and insecure connection - creds := NewHTTPSCreds("test", "test", string(certData), string(keyData), false) - client := GetRepoHTTPClient("https://localhost:9443/foo/bar", false, creds) + creds := NewHTTPSCreds("test", "test", string(certData), string(keyData), false, "http://proxy:5000") + client := GetRepoHTTPClient("https://localhost:9443/foo/bar", false, creds, "http://proxy:5000") assert.NotNil(t, client) assert.NotNil(t, client.Transport) if client.Transport != nil { @@ -166,11 +166,19 @@ func TestCustomHTTPClient(t *testing.T) { assert.NotNil(t, cert.PrivateKey) } } + proxy, err := httpClient.Proxy(nil) + assert.Nil(t, err) + assert.Equal(t, "http://proxy:5000", proxy.String()) } + os.Setenv("http_proxy", "http://proxy-from-env:7878") + defer func() { + assert.Nil(t, os.Unsetenv("http_proxy")) + }() + // Get HTTPSCreds without client cert creds, but insecure connection - creds = NewHTTPSCreds("test", "test", "", "", true) - client = GetRepoHTTPClient("https://localhost:9443/foo/bar", true, creds) + creds = NewHTTPSCreds("test", "test", "", "", true, "") + client = GetRepoHTTPClient("https://localhost:9443/foo/bar", true, creds, "") assert.NotNil(t, client) assert.NotNil(t, client.Transport) if client.Transport != nil { @@ -189,11 +197,16 @@ func TestCustomHTTPClient(t *testing.T) { assert.Nil(t, cert.PrivateKey) } } + req, err := http.NewRequest("GET", "http://proxy-from-env:7878", nil) + assert.Nil(t, err) + proxy, err := httpClient.Proxy(req) + assert.Nil(t, err) + assert.Equal(t, "http://proxy-from-env:7878", proxy.String()) } } func TestLsRemote(t *testing.T) { - clnt, err := NewClientExt("https://github.com/argoproj/argo-cd.git", "/tmp", NopCreds{}, false, false) + clnt, err := NewClientExt("https://github.com/argoproj/argo-cd.git", "/tmp", NopCreds{}, false, false, "") assert.NoError(t, err) xpass := []string{ "HEAD", @@ -238,7 +251,7 @@ func TestLFSClient(t *testing.T) { defer func() { _ = os.RemoveAll(tempDir) }() } - client, err := NewClientExt("https://github.com/argoproj-labs/argocd-testrepo-lfs", tempDir, NopCreds{}, false, true) + client, err := NewClientExt("https://github.com/argoproj-labs/argocd-testrepo-lfs", tempDir, NopCreds{}, false, true, "") assert.NoError(t, err) commitSHA, err := client.LsRemote("HEAD") @@ -248,7 +261,7 @@ func TestLFSClient(t *testing.T) { err = client.Init() assert.NoError(t, err) - err = client.Fetch() + err = client.Fetch("") assert.NoError(t, err) err = client.Checkout(commitSHA) @@ -271,22 +284,19 @@ func TestLFSClient(t *testing.T) { } func TestVerifyCommitSignature(t *testing.T) { - if os.Getenv("GNUPG_DISABLED") == "true" { - t.Skip() - } p, err := ioutil.TempDir("", "test-verify-commit-sig") if err != nil { panic(err.Error()) } defer os.RemoveAll(p) - client, err := NewClientExt("https://github.com/argoproj/argo-cd.git", p, NopCreds{}, false, false) + client, err := NewClientExt("https://github.com/argoproj/argo-cd.git", p, NopCreds{}, false, false, "") assert.NoError(t, err) err = client.Init() assert.NoError(t, err) - err = client.Fetch() + err = client.Fetch("") assert.NoError(t, err) commitSHA, err := client.LsRemote("HEAD") @@ -317,7 +327,6 @@ func TestNewFactory(t *testing.T) { defer addBinDirToPath.Close() closer := log.Debug() defer closer() - type args struct { url string insecureIgnoreHostKey bool @@ -326,8 +335,7 @@ func TestNewFactory(t *testing.T) { name string args args }{ - {"Github", args{url: "https://github.com/argoproj/argocd-example-apps"}}, - {"Azure", args{url: "https://jsuen0437@dev.azure.com/jsuen0437/jsuen/_git/jsuen"}}, + {"GitHub", args{url: "https://github.com/argoproj/argocd-example-apps"}}, } for _, tt := range tests { @@ -339,7 +347,7 @@ func TestNewFactory(t *testing.T) { assert.NoError(t, err) defer func() { _ = os.RemoveAll(dirName) }() - client, err := NewClientExt(tt.args.url, dirName, NopCreds{}, tt.args.insecureIgnoreHostKey, false) + client, err := NewClientExt(tt.args.url, dirName, NopCreds{}, tt.args.insecureIgnoreHostKey, false, "") assert.NoError(t, err) commitSHA, err := client.LsRemote("HEAD") assert.NoError(t, err) @@ -347,11 +355,11 @@ func TestNewFactory(t *testing.T) { err = client.Init() assert.NoError(t, err) - err = client.Fetch() + err = client.Fetch("") assert.NoError(t, err) // Do a second fetch to make sure we can treat `already up-to-date` error as not an error - err = client.Fetch() + err = client.Fetch("") assert.NoError(t, err) err = client.Checkout(commitSHA) @@ -380,7 +388,7 @@ func TestListRevisions(t *testing.T) { defer os.RemoveAll(dir) repoURL := "https://github.com/argoproj/argo-cd.git" - client, err := NewClientExt(repoURL, dir, NopCreds{}, false, false) + client, err := NewClientExt(repoURL, dir, NopCreds{}, false, false, "") assert.NoError(t, err) lsResult, err := client.LsRefs() diff --git a/ext/git/mocks/Client.go b/ext/git/mocks/Client.go index 719907e..c0b8344 100644 --- a/ext/git/mocks/Client.go +++ b/ext/git/mocks/Client.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.7.4. DO NOT EDIT. +// Code generated by mockery v1.1.2. DO NOT EDIT. package mocks @@ -103,27 +103,13 @@ func (_m *Client) Config(username string, email string) error { return r0 } -// Fetch provides a mock function with given fields: -func (_m *Client) Fetch() error { - ret := _m.Called() - - var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// FetchRef provides a mock function with given fields: ref -func (_m *Client) FetchRef(ref string) error { - ret := _m.Called(ref) +// Fetch provides a mock function with given fields: revision +func (_m *Client) Fetch(revision string) error { + ret := _m.Called(revision) var r0 error if rf, ok := ret.Get(0).(func(string) error); ok { - r0 = rf(ref) + r0 = rf(revision) } else { r0 = ret.Error(0) } diff --git a/ext/git/mocks/Creds.go b/ext/git/mocks/Creds.go index 457c210..205d4df 100644 --- a/ext/git/mocks/Creds.go +++ b/ext/git/mocks/Creds.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.7.4. DO NOT EDIT. +// Code generated by mockery v1.1.2. DO NOT EDIT. package mocks diff --git a/ext/git/mocks/GenericHTTPSCreds.go b/ext/git/mocks/GenericHTTPSCreds.go new file mode 100644 index 0000000..6224de7 --- /dev/null +++ b/ext/git/mocks/GenericHTTPSCreds.go @@ -0,0 +1,88 @@ +// Code generated by mockery v1.1.2. DO NOT EDIT. + +package mocks + +import ( + io "io" + + mock "github.com/stretchr/testify/mock" +) + +// GenericHTTPSCreds is an autogenerated mock type for the GenericHTTPSCreds type +type GenericHTTPSCreds struct { + mock.Mock +} + +// Environ provides a mock function with given fields: +func (_m *GenericHTTPSCreds) Environ() (io.Closer, []string, error) { + ret := _m.Called() + + var r0 io.Closer + if rf, ok := ret.Get(0).(func() io.Closer); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(io.Closer) + } + } + + var r1 []string + if rf, ok := ret.Get(1).(func() []string); ok { + r1 = rf() + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).([]string) + } + } + + var r2 error + if rf, ok := ret.Get(2).(func() error); ok { + r2 = rf() + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// GetClientCertData provides a mock function with given fields: +func (_m *GenericHTTPSCreds) GetClientCertData() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// GetClientCertKey provides a mock function with given fields: +func (_m *GenericHTTPSCreds) GetClientCertKey() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// HasClientCert provides a mock function with given fields: +func (_m *GenericHTTPSCreds) HasClientCert() bool { + ret := _m.Called() + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} diff --git a/ext/git/mocks/gitRefCache.go b/ext/git/mocks/gitRefCache.go new file mode 100644 index 0000000..61d7123 --- /dev/null +++ b/ext/git/mocks/gitRefCache.go @@ -0,0 +1,41 @@ +// Code generated by mockery v1.1.2. DO NOT EDIT. + +package mocks + +import ( + plumbing "github.com/go-git/go-git/v5/plumbing" + mock "github.com/stretchr/testify/mock" +) + +// gitRefCache is an autogenerated mock type for the gitRefCache type +type gitRefCache struct { + mock.Mock +} + +// GetGitReferences provides a mock function with given fields: repo, references +func (_m *gitRefCache) GetGitReferences(repo string, references *[]*plumbing.Reference) error { + ret := _m.Called(repo, references) + + var r0 error + if rf, ok := ret.Get(0).(func(string, *[]*plumbing.Reference) error); ok { + r0 = rf(repo, references) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// SetGitReferences provides a mock function with given fields: repo, references +func (_m *gitRefCache) SetGitReferences(repo string, references []*plumbing.Reference) error { + ret := _m.Called(repo, references) + + var r0 error + if rf, ok := ret.Get(0).(func(string, []*plumbing.Reference) error); ok { + r0 = rf(repo, references) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/ext/git/ssh.go b/ext/git/ssh.go new file mode 100644 index 0000000..eb07a05 --- /dev/null +++ b/ext/git/ssh.go @@ -0,0 +1,59 @@ +package git + +import ( + "fmt" + + gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "golang.org/x/crypto/ssh" +) + +// List of all currently supported algorithms for SSH key exchange +// Unfortunately, crypto/ssh does not offer public constants or list for +// this. +var SupportedSSHKeyExchangeAlgorithms = []string{ + "diffie-hellman-group1-sha1", + "diffie-hellman-group14-sha1", + "ecdh-sha2-nistp256", + "ecdh-sha2-nistp384", + "ecdh-sha2-nistp521", + "curve25519-sha256@libssh.org", + "diffie-hellman-group-exchange-sha1", + "diffie-hellman-group-exchange-sha256", +} + +// List of default key exchange algorithms to use. We use those that are +// available by default, we can become more opinionated later on (when +// we support configuration of algorithms to use). +var DefaultSSHKeyExchangeAlgorithms = SupportedSSHKeyExchangeAlgorithms + +// PublicKeysWithOptions is an auth method for go-git's SSH client that +// inherits from PublicKeys, but provides the possibility to override +// some client options. +type PublicKeysWithOptions struct { + KexAlgorithms []string + gitssh.PublicKeys +} + +// Name returns the name of the auth method +func (a *PublicKeysWithOptions) Name() string { + return gitssh.PublicKeysName +} + +// String returns the configured user and auth method name as string +func (a *PublicKeysWithOptions) String() string { + return fmt.Sprintf("user: %s, name: %s", a.User, a.Name()) +} + +// ClientConfig returns a custom SSH client configuration +func (a *PublicKeysWithOptions) ClientConfig() (*ssh.ClientConfig, error) { + // Algorithms used for kex can be configured + var kexAlgos []string + if len(a.KexAlgorithms) > 0 { + kexAlgos = a.KexAlgorithms + } else { + kexAlgos = DefaultSSHKeyExchangeAlgorithms + } + config := ssh.Config{KeyExchanges: kexAlgos} + opts := &ssh.ClientConfig{Config: config, User: a.User, Auth: []ssh.AuthMethod{ssh.PublicKeys(a.Signer)}} + return a.SetHostKeyCallback(opts) +} diff --git a/ext/git/workaround.go b/ext/git/workaround.go index 5e89acf..c364c09 100644 --- a/ext/git/workaround.go +++ b/ext/git/workaround.go @@ -1,20 +1,20 @@ package git import ( - "gopkg.in/src-d/go-git.v4" - "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/transport" - "gopkg.in/src-d/go-git.v4/plumbing/transport/client" - "gopkg.in/src-d/go-git.v4/plumbing/transport/http" - "gopkg.in/src-d/go-git.v4/utils/ioutil" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/client" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/utils/ioutil" ) // Below is a workaround for https://github.com/src-d/go-git/issues/1177: the `github.com/src-d/go-git` does not support disable SSL cert verification is a single repo. // As workaround methods `newUploadPackSession`, `newClient` and `listRemote` were copied from https://github.com/src-d/go-git/blob/master/remote.go and modified to use // transport with InsecureSkipVerify flag is verification should be disabled. -func newUploadPackSession(url string, auth transport.AuthMethod, insecure bool, creds Creds) (transport.UploadPackSession, error) { - c, ep, err := newClient(url, insecure, creds) +func newUploadPackSession(url string, auth transport.AuthMethod, insecure bool, creds Creds, proxy string) (transport.UploadPackSession, error) { + c, ep, err := newClient(url, insecure, creds, proxy) if err != nil { return nil, err } @@ -22,28 +22,26 @@ func newUploadPackSession(url string, auth transport.AuthMethod, insecure bool, return c.NewUploadPackSession(ep, auth) } -func newClient(url string, insecure bool, creds Creds) (transport.Transport, *transport.Endpoint, error) { +func newClient(url string, insecure bool, creds Creds, proxy string) (transport.Transport, *transport.Endpoint, error) { ep, err := transport.NewEndpoint(url) if err != nil { return nil, nil, err } - var c transport.Transport - // For HTTPS repositories, we get a custom Transport. Everything else will - // be default. - if IsHTTPSURL(url) { - c = http.NewClient(GetRepoHTTPClient(url, insecure, creds)) - } else { - c, err = client.NewClient(ep) + if !IsHTTPSURL(url) && !IsHTTPURL(url) { + // use the default client for protocols other than HTTP/HTTPS + c, err := client.NewClient(ep) if err != nil { return nil, nil, err } + return c, ep, nil } - return c, ep, err + + return http.NewClient(GetRepoHTTPClient(url, insecure, creds, proxy)), ep, nil } -func listRemote(r *git.Remote, o *git.ListOptions, insecure bool, creds Creds) (rfs []*plumbing.Reference, err error) { - s, err := newUploadPackSession(r.Config().URLs[0], o.Auth, insecure, creds) +func listRemote(r *git.Remote, o *git.ListOptions, insecure bool, creds Creds, proxy string) (rfs []*plumbing.Reference, err error) { + s, err := newUploadPackSession(r.Config().URLs[0], o.Auth, insecure, creds, proxy) if err != nil { return nil, err } diff --git a/ext/git/writer.go b/ext/git/writer.go new file mode 100644 index 0000000..4dede58 --- /dev/null +++ b/ext/git/writer.go @@ -0,0 +1,118 @@ +package git + +import ( + "fmt" + "strings" + + "github.com/argoproj-labs/argocd-image-updater/pkg/log" +) + +// CommitOptions holds options for a git commit operation +type CommitOptions struct { + // CommitMessageText holds a short commit message (-m option) + CommitMessageText string + // CommitMessagePath holds the path to a file to be used for the commit message (-F option) + CommitMessagePath string + // SigningKey holds a GnuPG key ID used to sign the commit with (-S option) + SigningKey string + // SignOff specifies whether to sign-off a commit (-s option) + SignOff bool +} + +// Commit perfoms a git commit for the given pathSpec to the currently checked +// out branch. If pathSpec is empty, or the special value "*", all pending +// changes will be commited. If message is not the empty string, it will be +// used as the commit message, otherwise a default commit message will be used. +// If signingKey is not the empty string, commit will be signed with the given +// GPG key. +func (m *nativeGitClient) Commit(pathSpec string, opts *CommitOptions) error { + defaultCommitMsg := "Update parameters" + args := []string{"commit"} + if pathSpec == "" || pathSpec == "*" { + args = append(args, "-a") + } + if opts.SigningKey != "" { + args = append(args, "-S", opts.SigningKey) + } + if opts.SignOff { + args = append(args, "-s") + } + if opts.CommitMessageText != "" { + args = append(args, "-m", opts.CommitMessageText) + } else if opts.CommitMessagePath != "" { + args = append(args, "-F", opts.CommitMessagePath) + } else { + args = append(args, "-m", defaultCommitMsg) + } + + out, err := m.runCmd(args...) + if err != nil { + log.Errorf(out) + return err + } + + return nil +} + +// Branch creates a new target branch from a given source branch +func (m *nativeGitClient) Branch(sourceBranch string, targetBranch string) error { + if sourceBranch != "" { + _, err := m.runCmd("checkout", sourceBranch) + if err != nil { + return fmt.Errorf("could not checkout source branch: %v", err) + } + } + + _, err := m.runCmd("branch", targetBranch) + if err != nil { + return fmt.Errorf("could not create new branch: %v", err) + } + + return nil +} + +// Push pushes local changes to the remote branch. If force is true, will force +// the remote to accept the push. +func (m *nativeGitClient) Push(remote string, branch string, force bool) error { + args := []string{"push"} + if force { + args = append(args, "-f") + } + args = append(args, remote, branch) + err := m.runCredentialedCmd("git", args...) + if err != nil { + return fmt.Errorf("could not push %s to %s: %v", branch, remote, err) + } + return nil +} + +// Add adds a path spec to the repository +func (m *nativeGitClient) Add(path string) error { + return m.runCredentialedCmd("git", "add", path) +} + +// SymRefToBranch retrieves the branch name a symbolic ref points to +func (m *nativeGitClient) SymRefToBranch(symRef string) (string, error) { + output, err := m.runCmd("symbolic-ref", symRef) + if err != nil { + return "", fmt.Errorf("could not resolve symbolic ref '%s': %v", symRef, err) + } + if a := strings.SplitN(output, "refs/heads/", 2); len(a) == 2 { + return a[1], nil + } + return "", fmt.Errorf("no symbolic ref named '%s' could be found", symRef) +} + +// Config configures username and email address for the repository +func (m *nativeGitClient) Config(username string, email string) error { + _, err := m.runCmd("config", "user.name", username) + if err != nil { + return fmt.Errorf("could not set git username: %v", err) + } + _, err = m.runCmd("config", "user.email", email) + if err != nil { + return fmt.Errorf("could not set git email: %v", err) + } + + return nil +} |
