diff options
| author | jannfis <jann@mistrust.net> | 2020-12-20 19:15:04 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-12-20 19:15:04 +0100 |
| commit | 4a7c10bfba8b3de2ace87c8c1798d87e82dd8afd (patch) | |
| tree | 3c5dace9230870a9c9930823ea8070e250dd16aa /ext/git | |
| parent | 69fbb79d64a60ab07ce2f65e494d55965a3c0b8c (diff) | |
feat: Git write-back of parameters (#133)
* feat: Initial support for persisting changes in Git
Signed-off-by: jannfis <jann@mistrust.net>
* Fix unit test
Signed-off-by: jannfis <jann@mistrust.net>
* Fix up some stuff
Signed-off-by: jannfis <jann@mistrust.net>
* More tests and fix mocks
Signed-off-by: jannfis <jann@mistrust.net>
* Spellings
* Spellings
* Spellings
* Fix lint
Signed-off-by: jannfis <jann@mistrust.net>
* Disable GPG tests
* Typo
* Exclude ext code from tests
Diffstat (limited to 'ext/git')
| -rw-r--r-- | ext/git/client.go | 627 | ||||
| -rw-r--r-- | ext/git/creds.go | 181 | ||||
| -rw-r--r-- | ext/git/git.go | 95 | ||||
| -rw-r--r-- | ext/git/git_test.go | 396 | ||||
| -rw-r--r-- | ext/git/mocks/Client.go | 280 | ||||
| -rw-r--r-- | ext/git/testdata/certs/argocd-test-client.crt | 85 | ||||
| -rw-r--r-- | ext/git/testdata/certs/argocd-test-client.key | 28 | ||||
| -rw-r--r-- | ext/git/workaround.go | 75 |
8 files changed, 1767 insertions, 0 deletions
diff --git a/ext/git/client.go b/ext/git/client.go new file mode 100644 index 0000000..e9ed558 --- /dev/null +++ b/ext/git/client.go @@ -0,0 +1,627 @@ +package git + +import ( + "crypto/tls" + "fmt" + "math" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/config" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/transport" + githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" + ssh2 "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh" + "gopkg.in/src-d/go-git.v4/storage/memory" + + "github.com/argoproj/argo-cd/common" + certutil "github.com/argoproj/argo-cd/util/cert" + executil "github.com/argoproj/argo-cd/util/exec" + + "github.com/argoproj-labs/argocd-image-updater/pkg/log" +) + +type RevisionMetadata struct { + Author string + Date time.Time + Tags []string + Message string +} + +// this should match reposerver/repository/repository.proto/RefsList +type Refs struct { + Branches []string + Tags []string + // heads and remotes are also refs, but are not needed at this time. +} + +// Client is a generic git client interface +type Client interface { + Root() string + Init() error + Fetch() error + FetchRef(ref string) error + Checkout(revision string) error + LsRefs() (*Refs, error) + LsRemote(revision string) (string, error) + LsFiles(path string) ([]string, error) + LsLargeFiles() ([]string, error) + CommitSHA() (string, error) + RevisionMetadata(revision string) (*RevisionMetadata, error) + VerifyCommitSignature(string) (string, error) + Branch(sourceBranch, targetBranch string) error + Commit(pathSpec string, message string, signingKey string) error + Push(remote string, branch string, force bool) error +} + +// nativeGitClient implements Client interface using git CLI +type nativeGitClient struct { + // URL of the repository + repoURL string + // Root path of repository + root string + // Authenticator credentials for private repositories + creds Creds + // Whether to connect insecurely to repository, e.g. don't verify certificate + insecure bool + // Whether the repository is LFS enabled + enableLfs bool +} + +var ( + maxAttemptsCount = 1 +) + +func init() { + if countStr := os.Getenv(common.EnvGitAttemptsCount); countStr != "" { + if cnt, err := strconv.Atoi(countStr); err != nil { + panic(fmt.Sprintf("Invalid value in %s env variable: %v", common.EnvGitAttemptsCount, err)) + } else { + maxAttemptsCount = int(math.Max(float64(cnt), 1)) + } + } +} + +func NewClient(rawRepoURL string, creds Creds, insecure bool, enableLfs bool) (Client, error) { + root := filepath.Join(os.TempDir(), strings.Replace(NormalizeGitURL(rawRepoURL), "/", "_", -1)) + if root == os.TempDir() { + return nil, fmt.Errorf("Repository '%s' cannot be initialized, because its root would be system temp at %s", rawRepoURL, root) + } + return NewClientExt(rawRepoURL, root, creds, insecure, enableLfs) +} + +func NewClientExt(rawRepoURL string, root string, creds Creds, insecure bool, enableLfs bool) (Client, error) { + client := nativeGitClient{ + repoURL: rawRepoURL, + root: root, + creds: creds, + insecure: insecure, + enableLfs: enableLfs, + } + return &client, nil +} + +// Returns a HTTP client object suitable for go-git to use using the following +// pattern: +// - If insecure is true, always returns a client with certificate verification +// turned off. +// - If one or more custom certificates are stored for the repository, returns +// a client with those certificates in the list of root CAs used to verify +// the server's certificate. +// - Otherwise (and on non-fatal errors), a default HTTP client is returned. +func GetRepoHTTPClient(repoURL string, insecure bool, creds Creds) *http.Client { + // Default HTTP client + var customHTTPClient = &http.Client{ + // 15 second timeout + Timeout: 15 * time.Second, + // don't follow redirect + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + // Callback function to return any configured client certificate + // We never return err, but an empty cert instead. + clientCertFunc := func(req *tls.CertificateRequestInfo) (*tls.Certificate, error) { + var err error + cert := tls.Certificate{} + + // If we aren't called with HTTPSCreds, then we just return an empty cert + httpsCreds, ok := creds.(HTTPSCreds) + if !ok { + return &cert, nil + } + + // If the creds contain client certificate data, we return a TLS.Certificate + // populated with the cert and its key. + if httpsCreds.clientCertData != "" && httpsCreds.clientCertKey != "" { + cert, err = tls.X509KeyPair([]byte(httpsCreds.clientCertData), []byte(httpsCreds.clientCertKey)) + if err != nil { + log.Errorf("Could not load Client Certificate: %v", err) + return &cert, nil + } + } + + return &cert, nil + } + + if insecure { + customHTTPClient.Transport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + GetClientCertificate: clientCertFunc, + }, + DisableKeepAlives: true, + } + } else { + parsedURL, err := url.Parse(repoURL) + if err != nil { + return customHTTPClient + } + serverCertificatePem, err := certutil.GetCertificateForConnect(parsedURL.Host) + if err != nil { + return customHTTPClient + } else if len(serverCertificatePem) > 0 { + certPool := certutil.GetCertPoolFromPEMData(serverCertificatePem) + customHTTPClient.Transport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + GetClientCertificate: clientCertFunc, + }, + DisableKeepAlives: true, + } + } else { + // else no custom certificate stored. + customHTTPClient.Transport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{ + GetClientCertificate: clientCertFunc, + }, + DisableKeepAlives: true, + } + } + } + + return customHTTPClient +} + +func newAuth(repoURL string, creds Creds) (transport.AuthMethod, error) { + switch creds := creds.(type) { + case SSHCreds: + var sshUser string + if isSSH, user := IsSSHURL(repoURL); isSSH { + sshUser = user + } + signer, err := ssh.ParsePrivateKey([]byte(creds.sshPrivateKey)) + if err != nil { + return nil, err + } + auth := &ssh2.PublicKeys{User: sshUser, Signer: signer} + if creds.insecure { + auth.HostKeyCallback = ssh.InsecureIgnoreHostKey() + } else { + // Set up validation of SSH known hosts for using our ssh_known_hosts + // file. + auth.HostKeyCallback, err = knownhosts.New(certutil.GetSSHKnownHostsDataPath()) + if err != nil { + log.Errorf("Could not set-up SSH known hosts callback: %v", err) + } + } + return auth, nil + case HTTPSCreds: + auth := githttp.BasicAuth{Username: creds.username, Password: creds.password} + return &auth, nil + } + return nil, nil +} + +func (m *nativeGitClient) Root() string { + return m.root +} + +// Init initializes a local git repository and sets the remote origin +func (m *nativeGitClient) Init() error { + _, err := git.PlainOpen(m.root) + if err == nil { + return nil + } + if err != git.ErrRepositoryNotExists { + return err + } + log.Infof("Initializing %s to %s", m.repoURL, m.root) + _, err = executil.Run(exec.Command("rm", "-rf", m.root)) + if err != nil { + return fmt.Errorf("unable to clean repo at %s: %v", m.root, err) + } + err = os.MkdirAll(m.root, 0755) + if err != nil { + return err + } + repo, err := git.PlainInit(m.root, false) + if err != nil { + return err + } + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: git.DefaultRemoteName, + URLs: []string{m.repoURL}, + }) + return err +} + +// Returns true if the repository is LFS enabled +func (m *nativeGitClient) IsLFSEnabled() bool { + return m.enableLfs +} + +// Fetch fetches latest updates from origin +func (m *nativeGitClient) Fetch() error { + err := m.runCredentialedCmd("git", "fetch", "origin", "--tags", "--force") + // When we have LFS support enabled, check for large files and fetch them too. + if err == nil && m.IsLFSEnabled() { + largeFiles, err := m.LsLargeFiles() + if err == nil && len(largeFiles) > 0 { + err = m.runCredentialedCmd("git", "lfs", "fetch", "--all") + if err != nil { + return err + } + } + } + return err +} + +// Fetch fetches latest updates from origin +func (m *nativeGitClient) FetchRef(ref string) error { + err := m.runCredentialedCmd("git", "fetch", "--tags", "--force") + // When we have LFS support enabled, check for large files and fetch them too. + if err == nil && m.IsLFSEnabled() { + largeFiles, err := m.LsLargeFiles() + if err == nil && len(largeFiles) > 0 { + err = m.runCredentialedCmd("git", "lfs", "fetch", "--all") + if err != nil { + return err + } + } + } + return err +} + +// LsFiles lists the local working tree, including only files that are under source control +func (m *nativeGitClient) LsFiles(path string) ([]string, error) { + out, err := m.runCmd("ls-files", "--full-name", "-z", "--", path) + if err != nil { + return nil, err + } + // remove last element, which is blank regardless of whether we're using nullbyte or newline + ss := strings.Split(out, "\000") + return ss[:len(ss)-1], nil +} + +// LsLargeFiles lists all files that have references to LFS storage +func (m *nativeGitClient) LsLargeFiles() ([]string, error) { + out, err := m.runCmd("lfs", "ls-files", "-n") + if err != nil { + return nil, err + } + ss := strings.Split(out, "\n") + return ss, nil +} + +// Checkout checkout specified git sha +func (m *nativeGitClient) Checkout(revision string) error { + if revision == "" || revision == "HEAD" { + revision = "origin/HEAD" + } + if _, err := m.runCmd("checkout", "--force", revision); err != nil { + return err + } + // We must populate LFS content by using lfs checkout, if we have at least + // one LFS reference in the current revision. + if m.IsLFSEnabled() { + if largeFiles, err := m.LsLargeFiles(); err == nil { + if len(largeFiles) > 0 { + if _, err := m.runCmd("lfs", "checkout"); err != nil { + return err + } + } + } else { + return err + } + } + if _, err := os.Stat(m.root + "/.gitmodules"); !os.IsNotExist(err) { + if submoduleEnabled := os.Getenv(common.EnvGitSubmoduleEnabled); submoduleEnabled != "false" { + if err := m.runCredentialedCmd("git", "submodule", "update", "--init", "--recursive"); err != nil { + return err + } + } + } + if _, err := m.runCmd("clean", "-fdx"); err != nil { + return err + } + return nil +} + +func (m *nativeGitClient) getRefs() ([]*plumbing.Reference, error) { + repo, err := git.Init(memory.NewStorage(), nil) + if err != nil { + return nil, err + } + remote, err := repo.CreateRemote(&config.RemoteConfig{ + Name: git.DefaultRemoteName, + URLs: []string{m.repoURL}, + }) + if err != nil { + return nil, err + } + auth, err := newAuth(m.repoURL, m.creds) + if err != nil { + return nil, err + } + return listRemote(remote, &git.ListOptions{Auth: auth}, m.insecure, m.creds) +} + +func (m *nativeGitClient) LsRefs() (*Refs, error) { + refs, err := m.getRefs() + + if err != nil { + return nil, err + } + + sortedRefs := &Refs{ + Branches: []string{}, + Tags: []string{}, + } + + for _, revision := range refs { + if revision.Name().IsBranch() { + sortedRefs.Branches = append(sortedRefs.Branches, revision.Name().Short()) + } else if revision.Name().IsTag() { + sortedRefs.Tags = append(sortedRefs.Tags, revision.Name().Short()) + } + } + + log.Debugf("LsRefs resolved %d branches and %d tags on repository", len(sortedRefs.Branches), len(sortedRefs.Tags)) + + // Would prefer to sort by last modified date but that info does not appear to be available without resolving each ref + sort.Strings(sortedRefs.Branches) + sort.Strings(sortedRefs.Tags) + + return sortedRefs, nil +} + +// LsRemote resolves the commit SHA of a specific branch, tag, or HEAD. If the supplied revision +// does not resolve, and "looks" like a 7+ hexadecimal commit SHA, it return the revision string. +// Otherwise, it returns an error indicating that the revision could not be resolved. This method +// runs with in-memory storage and is safe to run concurrently, or to be run without a git +// repository locally cloned. +func (m *nativeGitClient) LsRemote(revision string) (res string, err error) { + for attempt := 0; attempt < maxAttemptsCount; attempt++ { + if res, err = m.lsRemote(revision); err == nil { + return + } + } + return +} + +func (m *nativeGitClient) lsRemote(revision string) (string, error) { + if IsCommitSHA(revision) { + return revision, nil + } + + refs, err := m.getRefs() + + if err != nil { + return "", err + } + if revision == "" { + revision = "HEAD" + } + // refToHash keeps a maps of remote refs to their hash + // (e.g. refs/heads/master -> a67038ae2e9cb9b9b16423702f98b41e36601001) + refToHash := make(map[string]string) + // refToResolve remembers ref name of the supplied revision if we determine the revision is a + // symbolic reference (like HEAD), in which case we will resolve it from the refToHash map + refToResolve := "" + for _, ref := range refs { + refName := ref.Name().String() + if refName != "HEAD" && !strings.HasPrefix(refName, "refs/heads/") && !strings.HasPrefix(refName, "refs/tags/") { + // ignore things like 'refs/pull/' 'refs/reviewable' + continue + } + hash := ref.Hash().String() + if ref.Type() == plumbing.HashReference { + refToHash[refName] = hash + } + //log.Debugf("%s\t%s", hash, refName) + if ref.Name().Short() == revision { + if ref.Type() == plumbing.HashReference { + log.Debugf("revision '%s' resolved to '%s'", revision, hash) + return hash, nil + } + if ref.Type() == plumbing.SymbolicReference { + refToResolve = ref.Target().String() + } + } + } + if refToResolve != "" { + // If refToResolve is non-empty, we are resolving symbolic reference (e.g. HEAD). + // It should exist in our refToHash map + if hash, ok := refToHash[refToResolve]; ok { + log.Debugf("symbolic reference '%s' (%s) resolved to '%s'", revision, refToResolve, hash) + return hash, nil + } + } + // We support the ability to use a truncated commit-SHA (e.g. first 7 characters of a SHA) + if IsTruncatedCommitSHA(revision) { + log.Debugf("revision '%s' assumed to be commit sha", revision) + return revision, nil + } + // If we get here, revision string had non hexadecimal characters (indicating its a branch, tag, + // or symbolic ref) and we were unable to resolve it to a commit SHA. + return "", fmt.Errorf("Unable to resolve '%s' to a commit SHA", revision) +} + +// CommitSHA returns current commit sha from `git rev-parse HEAD` +func (m *nativeGitClient) CommitSHA() (string, error) { + out, err := m.runCmd("rev-parse", "HEAD") + if err != nil { + return "", err + } + return strings.TrimSpace(out), nil +} + +// returns the meta-data for the commit +func (m *nativeGitClient) RevisionMetadata(revision string) (*RevisionMetadata, error) { + out, err := m.runCmd("show", "-s", "--format=%an <%ae>|%at|%B", revision) + if err != nil { + return nil, err + } + segments := strings.SplitN(out, "|", 3) + if len(segments) != 3 { + return nil, fmt.Errorf("expected 3 segments, got %v", segments) + } + author := segments[0] + authorDateUnixTimestamp, _ := strconv.ParseInt(segments[1], 10, 64) + message := strings.TrimSpace(segments[2]) + + out, err = m.runCmd("tag", "--points-at", revision) + if err != nil { + return nil, err + } + tags := strings.Fields(out) + + return &RevisionMetadata{author, time.Unix(authorDateUnixTimestamp, 0), tags, message}, nil +} + +// VerifyCommitSignature Runs verify-commit on a given revision and returns the output +func (m *nativeGitClient) VerifyCommitSignature(revision string) (string, error) { + out, err := m.runGnuPGWrapper("git-verify-wrapper.sh", revision) + if err != nil { + return "", err + } + return out, nil +} + +func (m *nativeGitClient) Commit(pathSpec string, message string, signingKey string) error { + defaultCommitMsg := "Update parameters" + args := []string{"commit"} + if pathSpec == "" || pathSpec == "*" { + args = append(args, "-a") + } + if signingKey != "" { + args = append(args, "-S", signingKey) + } + if message != "" { + args = append(args, "-m", message) + } else { + args = append(args, "-m", defaultCommitMsg) + } + + out, err := m.runCmd(args...) + if err != nil { + log.Errorf(out) + return err + } + + return nil +} + +// Branch creates a new target branch from a given source branch +func (m *nativeGitClient) Branch(sourceBranch string, targetBranch string) error { + if sourceBranch != "" { + _, err := m.runCmd("git", "checkout", sourceBranch) + if err != nil { + return fmt.Errorf("could not checkout source branch: %v", err) + } + } + + _, err := m.runCmd("git", "branch", targetBranch) + if err != nil { + return fmt.Errorf("could not create new branch: %v", err) + } + + return nil +} + +func (m *nativeGitClient) Push(remote string, branch string, force bool) error { + args := []string{"push"} + if force { + args = append(args, "-f") + } + args = append(args, remote, branch) + err := m.runCredentialedCmd("git", args...) + if err != nil { + return fmt.Errorf("could not push %s to %s: %v", branch, remote, err) + } + return nil +} + +// runWrapper runs a custom command with all the semantics of running the Git client +func (m *nativeGitClient) runGnuPGWrapper(wrapper string, args ...string) (string, error) { + cmd := exec.Command(wrapper, args...) + cmd.Env = append(cmd.Env, fmt.Sprintf("GNUPGHOME=%s", common.GetGnuPGHomePath())) + return m.runCmdOutput(cmd) +} + +// runCmd is a convenience function to run a command in a given directory and return its output +func (m *nativeGitClient) runCmd(args ...string) (string, error) { + cmd := exec.Command("git", args...) + return m.runCmdOutput(cmd) +} + +// runCredentialedCmd is a convenience function to run a git command with username/password credentials +// nolint:unparam +func (m *nativeGitClient) runCredentialedCmd(command string, args ...string) error { + cmd := exec.Command(command, args...) + closer, environ, err := m.creds.Environ() + if err != nil { + return err + } + defer func() { _ = closer.Close() }() + cmd.Env = append(cmd.Env, environ...) + _, err = m.runCmdOutput(cmd) + return err +} + +func (m *nativeGitClient) runCmdOutput(cmd *exec.Cmd) (string, error) { + cmd.Dir = m.root + cmd.Env = append(cmd.Env, os.Environ()...) + // Set $HOME to nowhere, so we can be execute Git regardless of any external + // authentication keys (e.g. in ~/.ssh) -- this is especially important for + // running tests on local machines and/or CircleCI. + cmd.Env = append(cmd.Env, "HOME=/dev/null") + // Skip LFS for most Git operations except when explicitly requested + cmd.Env = append(cmd.Env, "GIT_LFS_SKIP_SMUDGE=1") + + // For HTTPS repositories, we need to consider insecure repositories as well + // as custom CA bundles from the cert database. + if IsHTTPSURL(m.repoURL) { + if m.insecure { + cmd.Env = append(cmd.Env, "GIT_SSL_NO_VERIFY=true") + } else { + parsedURL, err := url.Parse(m.repoURL) + // We don't fail if we cannot parse the URL, but log a warning in that + // case. And we execute the command in a verbatim way. + if err != nil { + log.Warnf("runCmdOutput: Could not parse repo URL '%s'", m.repoURL) + } else { + caPath, err := certutil.GetCertBundlePathForRepository(parsedURL.Host) + if err == nil && caPath != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_SSL_CAINFO=%s", caPath)) + } + } + } + } + return executil.Run(cmd) +} diff --git a/ext/git/creds.go b/ext/git/creds.go new file mode 100644 index 0000000..c869430 --- /dev/null +++ b/ext/git/creds.go @@ -0,0 +1,181 @@ +package git + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "strings" + + argoio "github.com/argoproj/gitops-engine/pkg/utils/io" + log "github.com/sirupsen/logrus" + + certutil "github.com/argoproj/argo-cd/util/cert" +) + +type Creds interface { + Environ() (io.Closer, []string, error) +} + +// nop implementation +type NopCloser struct { +} + +func (c NopCloser) Close() error { + return nil +} + +type NopCreds struct { +} + +func (c NopCreds) Environ() (io.Closer, []string, error) { + return NopCloser{}, nil, nil +} + +// HTTPS creds implementation +type HTTPSCreds struct { + // Username for authentication + username string + // Password for authentication + password string + // Whether to ignore invalid server certificates + insecure bool + // Client certificate to use + clientCertData string + // Client certificate key to use + clientCertKey string +} + +func NewHTTPSCreds(username string, password string, clientCertData string, clientCertKey string, insecure bool) HTTPSCreds { + return HTTPSCreds{ + username, + password, + insecure, + clientCertData, + clientCertKey, + } +} + +// Get additional required environment variables for executing git client to +// access specific repository via HTTPS. +func (c HTTPSCreds) Environ() (io.Closer, []string, error) { + env := []string{fmt.Sprintf("GIT_ASKPASS=%s", "git-ask-pass.sh"), fmt.Sprintf("GIT_USERNAME=%s", c.username), fmt.Sprintf("GIT_PASSWORD=%s", c.password)} + httpCloser := authFilePaths(make([]string, 0)) + + // GIT_SSL_NO_VERIFY is used to tell git not to validate the server's cert at + // all. + if c.insecure { + env = append(env, "GIT_SSL_NO_VERIFY=true") + } + + // In case the repo is configured for using a TLS client cert, we need to make + // sure git client will use it. The certificate's key must not be password + // protected. + if c.clientCertData != "" && c.clientCertKey != "" { + var certFile, keyFile *os.File + + // We need to actually create two temp files, one for storing cert data and + // another for storing the key. If we fail to create second fail, the first + // must be removed. + certFile, err := ioutil.TempFile(argoio.TempDir, "") + if err == nil { + defer certFile.Close() + keyFile, err = ioutil.TempFile(argoio.TempDir, "") + if err != nil { + removeErr := os.Remove(certFile.Name()) + if removeErr != nil { + log.Errorf("Could not remove previously created tempfile %s: %v", certFile.Name(), removeErr) + } + return NopCloser{}, nil, err + } + defer keyFile.Close() + } else { + return NopCloser{}, nil, err + } + + // We should have both temp files by now + httpCloser = authFilePaths([]string{certFile.Name(), keyFile.Name()}) + + _, err = certFile.WriteString(c.clientCertData) + if err != nil { + httpCloser.Close() + return NopCloser{}, nil, err + } + // GIT_SSL_CERT is the full path to a client certificate to be used + env = append(env, fmt.Sprintf("GIT_SSL_CERT=%s", certFile.Name())) + + _, err = keyFile.WriteString(c.clientCertKey) + if err != nil { + httpCloser.Close() + return NopCloser{}, nil, err + } + // GIT_SSL_KEY is the full path to a client certificate's key to be used + env = append(env, fmt.Sprintf("GIT_SSL_KEY=%s", keyFile.Name())) + + } + return httpCloser, env, nil +} + +// SSH implementation +type SSHCreds struct { + sshPrivateKey string + caPath string + insecure bool +} + +func NewSSHCreds(sshPrivateKey string, caPath string, insecureIgnoreHostKey bool) SSHCreds { + return SSHCreds{sshPrivateKey, caPath, insecureIgnoreHostKey} +} + +type sshPrivateKeyFile string + +type authFilePaths []string + +func (f sshPrivateKeyFile) Close() error { + return os.Remove(string(f)) +} + +// Remove a list of files that have been created as temp files while creating +// HTTPCreds object above. +func (f authFilePaths) Close() error { + var retErr error = nil + for _, path := range f { + err := os.Remove(path) + if err != nil { + log.Errorf("HTTPSCreds.Close(): Could not remove temp file %s: %v", path, err) + retErr = err + } + } + return retErr +} + +func (c SSHCreds) Environ() (io.Closer, []string, error) { + // use the SHM temp dir from util, more secure + file, err := ioutil.TempFile(argoio.TempDir, "") + if err != nil { + return nil, nil, err + } + defer file.Close() + + _, err = file.WriteString(c.sshPrivateKey + "\n") + if err != nil { + return nil, nil, err + } + + args := []string{"ssh", "-i", file.Name()} + var env []string + if c.caPath != "" { + env = append(env, fmt.Sprintf("GIT_SSL_CAINFO=%s", c.caPath)) + } + if c.insecure { + log.Warn("temporarily disabling strict host key checking (i.e. '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'), please don't use in production") + // StrictHostKeyChecking will add the host to the knownhosts file, we don't want that - a security issue really, + // UserKnownHostsFile=/dev/null is therefore used so we write the new insecure host to /dev/null + args = append(args, "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null") + } else { + knownHostsFile := certutil.GetSSHKnownHostsDataPath() + args = append(args, "-o", "StrictHostKeyChecking=yes", "-o", fmt.Sprintf("UserKnownHostsFile=%s", knownHostsFile)) + } + env = append(env, []string{fmt.Sprintf("GIT_SSH_COMMAND=%s", strings.Join(args, " "))}...) + return sshPrivateKeyFile(file.Name()), env, nil +} diff --git a/ext/git/git.go b/ext/git/git.go new file mode 100644 index 0000000..64d3587 --- /dev/null +++ b/ext/git/git.go @@ -0,0 +1,95 @@ +package git + +import ( + "net/url" + "regexp" + "strings" +) + +// EnsurePrefix idempotently ensures that a base string has a given prefix. +func ensurePrefix(s, prefix string) string { + if !strings.HasPrefix(s, prefix) { + s = prefix + s + } + return s +} + +// removeSuffix idempotently removes a given suffix +func removeSuffix(s, suffix string) string { + if strings.HasSuffix(s, suffix) { + return s[0 : len(s)-len(suffix)] + } + return s +} + +var ( + commitSHARegex = regexp.MustCompile("^[0-9A-Fa-f]{40}$") + sshURLRegex = regexp.MustCompile("^(ssh://)?([^/:]*?)@[^@]+$") + httpsURLRegex = regexp.MustCompile("^(https://).*") +) + +// IsCommitSHA returns whether or not a string is a 40 character SHA-1 +func IsCommitSHA(sha string) bool { + return commitSHARegex.MatchString(sha) +} + +var truncatedCommitSHARegex = regexp.MustCompile("^[0-9A-Fa-f]{7,}$") + +// IsTruncatedCommitSHA returns whether or not a string is a truncated SHA-1 +func IsTruncatedCommitSHA(sha string) bool { + return truncatedCommitSHARegex.MatchString(sha) +} + +// SameURL returns whether or not the two repository URLs are equivalent in location +func SameURL(leftRepo, rightRepo string) bool { + normalLeft := NormalizeGitURL(leftRepo) + normalRight := NormalizeGitURL(rightRepo) + return normalLeft != "" && normalRight != "" && normalLeft == normalRight +} + +// NormalizeGitURL normalizes a git URL for purposes of comparison, as well as preventing redundant +// local clones (by normalizing various forms of a URL to a consistent location). +// Prefer using SameURL() over this function when possible. This algorithm may change over time +// and should not be considered stable from release to release +func NormalizeGitURL(repo string) string { + repo = strings.ToLower(strings.TrimSpace(repo)) + if yes, _ := IsSSHURL(repo); yes { + if !strings.HasPrefix(repo, "ssh://") { + // We need to replace the first colon in git@server... style SSH URLs with a slash, otherwise + // net/url.Parse will interpret it incorrectly as the port. + repo = strings.Replace(repo, ":", "/", 1) + repo = ensurePrefix(repo, "ssh://") + } + } + repo = removeSuffix(repo, ".git") + repoURL, err := url.Parse(repo) + if err != nil { + return "" + } + normalized := repoURL.String() + return strings.TrimPrefix(normalized, "ssh://") +} + +// IsSSHURL returns true if supplied URL is SSH URL +func IsSSHURL(url string) (bool, string) { + matches := sshURLRegex.FindStringSubmatch(url) + if len(matches) > 2 { + return true, matches[2] + } + return false, "" +} + +// IsHTTPSURL returns true if supplied URL is HTTPS URL +func IsHTTPSURL(url string) bool { + return httpsURLRegex.MatchString(url) +} + +// TestRepo tests if a repo exists and is accessible with the given credentials +func TestRepo(repo string, creds Creds, insecure bool, enableLfs bool) error { + clnt, err := NewClient(repo, creds, insecure, enableLfs) + if err != nil { + return err + } + _, err = clnt.LsRemote("HEAD") + return err +} diff --git a/ext/git/git_test.go b/ext/git/git_test.go new file mode 100644 index 0000000..8d09646 --- /dev/null +++ b/ext/git/git_test.go @@ -0,0 +1,396 @@ +package git + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/argoproj/argo-cd/test/fixture/log" + "github.com/argoproj/argo-cd/test/fixture/path" + "github.com/argoproj/argo-cd/test/fixture/test" +) + +func TestIsCommitSHA(t *testing.T) { + assert.True(t, IsCommitSHA("9d921f65f3c5373b682e2eb4b37afba6592e8f8b")) + assert.True(t, IsCommitSHA("9D921F65F3C5373B682E2EB4B37AFBA6592E8F8B")) + assert.False(t, IsCommitSHA("gd921f65f3c5373b682e2eb4b37afba6592e8f8b")) + assert.False(t, IsCommitSHA("master")) + assert.False(t, IsCommitSHA("HEAD")) + assert.False(t, IsCommitSHA("9d921f6")) // only consider 40 characters hex strings as a commit-sha + assert.True(t, IsTruncatedCommitSHA("9d921f6")) + assert.False(t, IsTruncatedCommitSHA("9d921f")) // we only consider 7+ characters + assert.False(t, IsTruncatedCommitSHA("branch-name")) +} + +func TestEnsurePrefix(t *testing.T) { + data := [][]string{ + {"world", "hello", "helloworld"}, + {"helloworld", "hello", "helloworld"}, + {"example.com", "https://", "https://example.com"}, + {"https://example.com", "https://", "https://example.com"}, + {"cd", "argo", "argocd"}, + {"argocd", "argo", "argocd"}, + {"", "argocd", "argocd"}, + {"argocd", "", "argocd"}, + } + for _, table := range data { + result := ensurePrefix(table[0], table[1]) + assert.Equal(t, table[2], result) + } +} + +func TestRemoveSuffix(t *testing.T) { + data := [][]string{ + {"hello.git", ".git", "hello"}, + {"hello", ".git", "hello"}, + {".git", ".git", ""}, + } + for _, table := range data { + result := removeSuffix(table[0], table[1]) + assert.Equal(t, table[2], result) + } +} + +func TestIsSSHURL(t *testing.T) { + data := map[string]bool{ + "git://github.com/argoproj/test.git": false, + "git@GITHUB.com:argoproj/test.git": true, + "git@github.com:test": true, + "git@github.com:test.git": true, + "https://github.com/argoproj/test": false, + "https://github.com/argoproj/test.git": false, + "ssh://git@GITHUB.com:argoproj/test": true, + "ssh://git@GITHUB.com:argoproj/test.git": true, + "ssh://git@github.com:test.git": true, + } + for k, v := range data { + isSSH, _ := IsSSHURL(k) + assert.Equal(t, v, isSSH) + } +} + +func TestIsSSHURLUserName(t *testing.T) { + isSSH, user := IsSSHURL("ssh://john@john-server.org:29418/project") + assert.True(t, isSSH) + assert.Equal(t, "john", user) + + isSSH, user = IsSSHURL("john@john-server.org:29418/project") + assert.True(t, isSSH) + assert.Equal(t, "john", user) + + isSSH, user = IsSSHURL("john@doe.org@john-server.org:29418/project") + assert.True(t, isSSH) + assert.Equal(t, "john@doe.org", user) + + isSSH, user = IsSSHURL("ssh://john@doe.org@john-server.org:29418/project") + assert.True(t, isSSH) + assert.Equal(t, "john@doe.org", user) + + isSSH, user = IsSSHURL("john@doe.org@john-server.org:project") + assert.True(t, isSSH) + assert.Equal(t, "john@doe.org", user) + + isSSH, user = IsSSHURL("john@doe.org@john-server.org:29418/project") + assert.True(t, isSSH) + assert.Equal(t, "john@doe.org", user) + +} + +func TestSameURL(t *testing.T) { + data := map[string]string{ + "git@GITHUB.com:argoproj/test": "git@github.com:argoproj/test.git", + "git@GITHUB.com:argoproj/test.git": "git@github.com:argoproj/test.git", + "git@GITHUB.com:test": "git@github.com:test.git", + "git@GITHUB.com:test.git": "git@github.com:test.git", + "https://GITHUB.com/argoproj/test": "https://github.com/argoproj/test.git", + "https://GITHUB.com/argoproj/test.git": "https://github.com/argoproj/test.git", + "https://github.com/FOO": "https://github.com/foo", + "https://github.com/TEST": "https://github.com/TEST.git", + "https://github.com/TEST.git": "https://github.com/TEST.git", + "https://github.com:4443/TEST": "https://github.com:4443/TEST.git", + "https://github.com:4443/TEST.git": "https://github.com:4443/TEST", + "ssh://git@GITHUB.com/argoproj/test": "git@github.com:argoproj/test.git", + "ssh://git@GITHUB.com/argoproj/test.git": "git@github.com:argoproj/test.git", + "ssh://git@GITHUB.com/test.git": "git@github.com:test.git", + "ssh://git@github.com/test": "git@github.com:test.git", + " https://github.com/argoproj/test ": "https://github.com/argoproj/test.git", + "\thttps://github.com/argoproj/test\n": "https://github.com/argoproj/test.git", + "https://1234.visualstudio.com/myproj/_git/myrepo": "https://1234.visualstudio.com/myproj/_git/myrepo", + "https://dev.azure.com/1234/myproj/_git/myrepo": "https://dev.azure.com/1234/myproj/_git/myrepo", + } + for k, v := range data { + assert.True(t, SameURL(k, v)) + } +} + +func TestCustomHTTPClient(t *testing.T) { + certFile, err := filepath.Abs("testdata/certs/argocd-test-client.crt") + assert.NoError(t, err) + assert.NotEqual(t, "", certFile) + + keyFile, err := filepath.Abs("testdata/certs/argocd-test-client.key") + assert.NoError(t, err) + assert.NotEqual(t, "", keyFile) + + certData, err := ioutil.ReadFile(certFile) + assert.NoError(t, err) + assert.NotEqual(t, "", string(certData)) + + keyData, err := ioutil.ReadFile(keyFile) + assert.NoError(t, err) + assert.NotEqual(t, "", string(keyData)) + + // Get HTTPSCreds with client cert creds specified, and insecure connection + creds := NewHTTPSCreds("test", "test", string(certData), string(keyData), false) + client := GetRepoHTTPClient("https://localhost:9443/foo/bar", false, creds) + assert.NotNil(t, client) + assert.NotNil(t, client.Transport) + if client.Transport != nil { + httpClient := client.Transport.(*http.Transport) + assert.NotNil(t, httpClient.TLSClientConfig) + + assert.Equal(t, false, httpClient.TLSClientConfig.InsecureSkipVerify) + + assert.NotNil(t, httpClient.TLSClientConfig.GetClientCertificate) + if httpClient.TLSClientConfig.GetClientCertificate != nil { + cert, err := httpClient.TLSClientConfig.GetClientCertificate(nil) + assert.NoError(t, err) + if err == nil { + assert.NotNil(t, cert) + assert.NotEqual(t, 0, len(cert.Certificate)) + assert.NotNil(t, cert.PrivateKey) + } + } + } + + // Get HTTPSCreds without client cert creds, but insecure connection + creds = NewHTTPSCreds("test", "test", "", "", true) + client = GetRepoHTTPClient("https://localhost:9443/foo/bar", true, creds) + assert.NotNil(t, client) + assert.NotNil(t, client.Transport) + if client.Transport != nil { + httpClient := client.Transport.(*http.Transport) + assert.NotNil(t, httpClient.TLSClientConfig) + + assert.Equal(t, true, httpClient.TLSClientConfig.InsecureSkipVerify) + + assert.NotNil(t, httpClient.TLSClientConfig.GetClientCertificate) + if httpClient.TLSClientConfig.GetClientCertificate != nil { + cert, err := httpClient.TLSClientConfig.GetClientCertificate(nil) + assert.NoError(t, err) + if err == nil { + assert.NotNil(t, cert) + assert.Equal(t, 0, len(cert.Certificate)) + assert.Nil(t, cert.PrivateKey) + } + } + } +} + +func TestLsRemote(t *testing.T) { + clnt, err := NewClientExt("https://github.com/argoproj/argo-cd.git", "/tmp", NopCreds{}, false, false) + assert.NoError(t, err) + xpass := []string{ + "HEAD", + "master", + "release-0.8", + "v0.8.0", + "4e22a3cb21fa447ca362a05a505a69397c8a0d44", + //"4e22a3c", + } + for _, revision := range xpass { + commitSHA, err := clnt.LsRemote(revision) + assert.NoError(t, err) + assert.True(t, IsCommitSHA(commitSHA)) + } + + // We do not resolve truncated git hashes and return the commit as-is if it appears to be a commit + commitSHA, err := clnt.LsRemote("4e22a3c") + assert.NoError(t, err) + assert.False(t, IsCommitSHA(commitSHA)) + assert.True(t, IsTruncatedCommitSHA(commitSHA)) + + xfail := []string{ + "unresolvable", + "4e22a3", // too short (6 characters) + } + for _, revision := range xfail { + _, err := clnt.LsRemote(revision) + assert.Error(t, err) + } +} + +// Running this test requires git-lfs to be installed on your machine. +func TestLFSClient(t *testing.T) { + + // temporary disable LFS test + // TODO(alexmt): dockerize tests in and enabled it + t.Skip() + + tempDir, err := ioutil.TempDir("", "git-client-lfs-test-") + assert.NoError(t, err) + if err == nil { + defer func() { _ = os.RemoveAll(tempDir) }() + } + + client, err := NewClientExt("https://github.com/argoproj-labs/argocd-testrepo-lfs", tempDir, NopCreds{}, false, true) + assert.NoError(t, err) + + commitSHA, err := client.LsRemote("HEAD") + assert.NoError(t, err) + assert.NotEqual(t, "", commitSHA) + + err = client.Init() + assert.NoError(t, err) + + err = client.Fetch() + assert.NoError(t, err) + + err = client.Checkout(commitSHA) + assert.NoError(t, err) + + largeFiles, err := client.LsLargeFiles() + assert.NoError(t, err) + assert.Equal(t, 3, len(largeFiles)) + + fileHandle, err := os.Open(fmt.Sprintf("%s/test3.yaml", tempDir)) + assert.NoError(t, err) + if err == nil { + defer fileHandle.Close() + text, err := ioutil.ReadAll(fileHandle) + assert.NoError(t, err) + if err == nil { + assert.Equal(t, "This is not a YAML, sorry.\n", string(text)) + } + } +} + +func TestVerifyCommitSignature(t *testing.T) { + if os.Getenv("GNUPG_DISABLED") == "true" { + t.Skip() + } + p, err := ioutil.TempDir("", "test-verify-commit-sig") + if err != nil { + panic(err.Error()) + } + defer os.RemoveAll(p) + + client, err := NewClientExt("https://github.com/argoproj/argo-cd.git", p, NopCreds{}, false, false) + assert.NoError(t, err) + + err = client.Init() + assert.NoError(t, err) + + err = client.Fetch() + assert.NoError(t, err) + + commitSHA, err := client.LsRemote("HEAD") + assert.NoError(t, err) + + err = client.Checkout(commitSHA) + assert.NoError(t, err) + + // 28027897aad1262662096745f2ce2d4c74d02b7f is a commit that is signed in the repo + // It doesn't matter whether we know the key or not at this stage + { + out, err := client.VerifyCommitSignature("28027897aad1262662096745f2ce2d4c74d02b7f") + assert.NoError(t, err) + assert.NotEmpty(t, out) + assert.Contains(t, out, "gpg: Signature made") + } + + // 85d660f0b967960becce3d49bd51c678ba2a5d24 is a commit that is not signed + { + out, err := client.VerifyCommitSignature("85d660f0b967960becce3d49bd51c678ba2a5d24") + assert.NoError(t, err) + assert.Empty(t, out) + } +} + +func TestNewFactory(t *testing.T) { + addBinDirToPath := path.NewBinDirToPath() + defer addBinDirToPath.Close() + closer := log.Debug() + defer closer() + + type args struct { + url string + insecureIgnoreHostKey bool + } + tests := []struct { + name string + args args + }{ + {"Github", args{url: "https://github.com/argoproj/argocd-example-apps"}}, + {"Azure", args{url: "https://jsuen0437@dev.azure.com/jsuen0437/jsuen/_git/jsuen"}}, + } + for _, tt := range tests { + + if tt.name == "PrivateSSHRepo" { + test.Flaky(t) + } + + dirName, err := ioutil.TempDir("", "git-client-test-") + assert.NoError(t, err) + defer func() { _ = os.RemoveAll(dirName) }() + + client, err := NewClientExt(tt.args.url, dirName, NopCreds{}, tt.args.insecureIgnoreHostKey, false) + assert.NoError(t, err) + commitSHA, err := client.LsRemote("HEAD") + assert.NoError(t, err) + + err = client.Init() + assert.NoError(t, err) + + err = client.Fetch() + assert.NoError(t, err) + + // Do a second fetch to make sure we can treat `already up-to-date` error as not an error + err = client.Fetch() + assert.NoError(t, err) + + err = client.Checkout(commitSHA) + assert.NoError(t, err) + + revisionMetadata, err := client.RevisionMetadata(commitSHA) + assert.NoError(t, err) + assert.NotNil(t, revisionMetadata) + assert.Regexp(t, "^.*<.*>$", revisionMetadata.Author) + assert.Len(t, revisionMetadata.Tags, 0) + assert.NotEmpty(t, revisionMetadata.Date) + assert.NotEmpty(t, revisionMetadata.Message) + + commitSHA2, err := client.CommitSHA() + assert.NoError(t, err) + + assert.Equal(t, commitSHA, commitSHA2) + } +} + +func TestListRevisions(t *testing.T) { + dir, err := ioutil.TempDir("", "test-list-revisions") + if err != nil { + panic(err.Error()) + } + defer os.RemoveAll(dir) + + repoURL := "https://github.com/argoproj/argo-cd.git" + client, err := NewClientExt(repoURL, dir, NopCreds{}, false, false) + assert.NoError(t, err) + + lsResult, err := client.LsRefs() + assert.NoError(t, err) + + testBranch := "master" + testTag := "v1.0.0" + + assert.Contains(t, lsResult.Branches, testBranch) + assert.Contains(t, lsResult.Tags, testTag) + assert.NotContains(t, lsResult.Branches, testTag) + assert.NotContains(t, lsResult.Tags, testBranch) +} diff --git a/ext/git/mocks/Client.go b/ext/git/mocks/Client.go new file mode 100644 index 0000000..05d4f5d --- /dev/null +++ b/ext/git/mocks/Client.go @@ -0,0 +1,280 @@ +// Code generated by mockery v1.1.2. DO NOT EDIT. + +package mocks + +import ( + git "github.com/argoproj-labs/argocd-image-updater/ext/git" + mock "github.com/stretchr/testify/mock" +) + +// Client is an autogenerated mock type for the Client type +type Client struct { + mock.Mock +} + +// Branch provides a mock function with given fields: sourceBranch, targetBranch +func (_m *Client) Branch(sourceBranch string, targetBranch string) error { + ret := _m.Called(sourceBranch, targetBranch) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string) error); ok { + r0 = rf(sourceBranch, targetBranch) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Checkout provides a mock function with given fields: revision +func (_m *Client) Checkout(revision string) error { + ret := _m.Called(revision) + + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(revision) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Commit provides a mock function with given fields: pathSpec, message, signingKey +func (_m *Client) Commit(pathSpec string, message string, signingKey string) error { + ret := _m.Called(pathSpec, message, signingKey) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, string) error); ok { + r0 = rf(pathSpec, message, signingKey) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// CommitSHA provides a mock function with given fields: +func (_m *Client) CommitSHA() (string, error) { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Fetch provides a mock function with given fields: +func (_m *Client) Fetch() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// FetchRef provides a mock function with given fields: ref +func (_m *Client) FetchRef(ref string) error { + ret := _m.Called(ref) + + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(ref) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Init provides a mock function with given fields: +func (_m *Client) Init() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// LsFiles provides a mock function with given fields: path +func (_m *Client) LsFiles(path string) ([]string, error) { + ret := _m.Called(path) + + var r0 []string + if rf, ok := ret.Get(0).(func(string) []string); ok { + r0 = rf(path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(path) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// LsLargeFiles provides a mock function with given fields: +func (_m *Client) LsLargeFiles() ([]string, error) { + ret := _m.Called() + + var r0 []string + if rf, ok := ret.Get(0).(func() []string); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// LsRefs provides a mock function with given fields: +func (_m *Client) LsRefs() (*git.Refs, error) { + ret := _m.Called() + + var r0 *git.Refs + if rf, ok := ret.Get(0).(func() *git.Refs); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*git.Refs) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// LsRemote provides a mock function with given fields: revision +func (_m *Client) LsRemote(revision string) (string, error) { + ret := _m.Called(revision) + + var r0 string + if rf, ok := ret.Get(0).(func(string) string); ok { + r0 = rf(revision) + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(revision) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Push provides a mock function with given fields: remote, branch, force +func (_m *Client) Push(remote string, branch string, force bool) error { + ret := _m.Called(remote, branch, force) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, bool) error); ok { + r0 = rf(remote, branch, force) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// RevisionMetadata provides a mock function with given fields: revision +func (_m *Client) RevisionMetadata(revision string) (*git.RevisionMetadata, error) { + ret := _m.Called(revision) + + var r0 *git.RevisionMetadata + if rf, ok := ret.Get(0).(func(string) *git.RevisionMetadata); ok { + r0 = rf(revision) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*git.RevisionMetadata) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(revision) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Root provides a mock function with given fields: +func (_m *Client) Root() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// VerifyCommitSignature provides a mock function with given fields: _a0 +func (_m *Client) VerifyCommitSignature(_a0 string) (string, error) { + ret := _m.Called(_a0) + + var r0 string + if rf, ok := ret.Get(0).(func(string) string); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/ext/git/testdata/certs/argocd-test-client.crt b/ext/git/testdata/certs/argocd-test-client.crt new file mode 100644 index 0000000..4a15070 --- /dev/null +++ b/ext/git/testdata/certs/argocd-test-client.crt @@ -0,0 +1,85 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 33:af:0d:00:3e:e7:90:bb:8c:77:0e:24:82:e3:9d:61 + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=ArgoCD Test CA + Validity + Not Before: Jul 20 15:39:35 2019 GMT + Not After : Jun 26 15:39:35 2119 GMT + Subject: CN=argo-test-client + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (2048 bit) + Modulus: + 00:96:1e:04:cc:35:14:87:98:83:58:87:21:89:2c: + 40:a6:40:25:60:6a:d8:85:fd:b0:c2:d7:06:8d:4a: + 0c:c3:ff:2a:98:77:60:f1:e6:74:49:b2:5d:8b:ef: + e3:80:0e:e3:6e:e0:50:7d:13:b8:fe:89:3a:ab:a5: + 05:23:6d:ba:6d:f8:3e:b3:75:90:c8:41:3d:40:c8: + 10:8d:62:a8:a9:4b:5a:6b:56:95:94:89:ca:28:f4: + 7a:a4:77:07:3a:35:15:b1:79:04:3a:43:74:70:51: + e7:9c:41:29:92:d3:f3:69:67:1a:1d:4a:be:0d:6e: + 72:bf:29:72:7d:1c:49:54:54:93:20:f3:7f:00:42: + e1:98:12:57:8e:39:a7:87:6f:f5:13:3c:25:be:9b: + 00:5f:57:07:c8:f4:f8:b8:ca:36:fd:29:35:49:70: + 31:66:0b:e7:a5:36:fb:10:ba:86:f9:18:98:17:d9: + 7d:e0:60:4f:cf:08:85:57:8d:8b:e0:19:fe:83:28: + 89:98:6f:2a:d5:85:ea:1e:59:0c:04:f5:87:b6:ff: + a2:8d:4c:62:ed:68:ba:9a:7f:2f:ac:94:c8:72:4e: + 24:f7:37:54:19:f5:14:65:3e:65:ff:7b:e4:f9:c1: + 42:80:a7:87:15:29:b9:78:26:f1:02:f2:4d:77:b8: + 04:79 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + X509v3 Subject Key Identifier: + AB:D1:69:33:0A:6A:F4:46:B1:80:D3:F7:B2:93:E8:07:FB:03:E8:BF + X509v3 Authority Key Identifier: + keyid:1E:B1:40:6A:1F:AB:D5:3D:A7:DF:07:AE:AB:5C:37:C3:CB:18:96:AC + DirName:/CN=ArgoCD Test CA + serial:06:04:D7:D4:4E:A1:2C:3F:A4:30:9E:AD:0C:A4:DF:55:68:F7:D3:C5 + + X509v3 Extended Key Usage: + TLS Web Client Authentication + X509v3 Key Usage: + Digital Signature + Signature Algorithm: sha256WithRSAEncryption + 03:a8:d4:3b:17:66:fc:93:20:b0:68:f9:79:fe:cd:e0:54:d6: + a4:20:fe:14:75:c9:63:f0:a3:ff:4a:a5:b8:d9:c8:43:fa:0b: + 9c:da:0b:fd:23:b9:cb:c0:9e:5a:db:72:21:9e:c5:56:81:32: + 14:4e:d5:ef:9b:97:ab:b8:93:1f:79:41:b0:fa:66:93:28:93: + 95:54:4a:8a:27:26:8a:fe:81:fd:a5:68:f2:9b:9c:6c:63:c3: + 96:98:9a:e9:e5:6d:34:69:f1:ea:ca:78:10:e4:2b:e1:41:bf: + dc:b6:c8:ba:76:ea:17:69:3e:cf:75:b8:28:03:17:06:0f:e5: + 9a:cb:36:27:85:d7:b8:13:92:69:1c:ce:72:fb:71:1f:38:a2: + 22:fa:86:13:20:44:79:77:9f:ab:11:e8:6e:65:94:b7:ee:c4: + 39:bd:89:45:4c:55:80:92:a6:83:83:83:75:3c:30:4e:da:6b: + 4b:74:0e:a7:86:4e:4f:79:d3:d2:a6:38:d5:ea:7d:fc:5f:a7: + 73:3b:97:ef:cb:49:08:10:96:13:64:68:48:d0:b3:eb:59:93: + 9d:22:ba:0c:83:1c:74:a1:f6:61:34:b6:8e:e5:e1:25:5a:09: + ec:7e:b0:b9:fd:21:7e:65:5d:3b:15:d7:a0:a3:e1:4f:fa:4e: + 77:90:3c:83 +-----BEGIN CERTIFICATE----- +MIIDZjCCAk6gAwIBAgIQM68NAD7nkLuMdw4kguOdYTANBgkqhkiG9w0BAQsFADAZ +MRcwFQYDVQQDDA5BcmdvQ0QgVGVzdCBDQTAgFw0xOTA3MjAxNTM5MzVaGA8yMTE5 +MDYyNjE1MzkzNVowGzEZMBcGA1UEAwwQYXJnby10ZXN0LWNsaWVudDCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBAJYeBMw1FIeYg1iHIYksQKZAJWBq2IX9 +sMLXBo1KDMP/Kph3YPHmdEmyXYvv44AO427gUH0TuP6JOqulBSNtum34PrN1kMhB +PUDIEI1iqKlLWmtWlZSJyij0eqR3Bzo1FbF5BDpDdHBR55xBKZLT82lnGh1Kvg1u +cr8pcn0cSVRUkyDzfwBC4ZgSV445p4dv9RM8Jb6bAF9XB8j0+LjKNv0pNUlwMWYL +56U2+xC6hvkYmBfZfeBgT88IhVeNi+AZ/oMoiZhvKtWF6h5ZDAT1h7b/oo1MYu1o +upp/L6yUyHJOJPc3VBn1FGU+Zf975PnBQoCnhxUpuXgm8QLyTXe4BHkCAwEAAaOB +pTCBojAJBgNVHRMEAjAAMB0GA1UdDgQWBBSr0WkzCmr0RrGA0/eyk+gH+wPovzBU +BgNVHSMETTBLgBQesUBqH6vVPaffB66rXDfDyxiWrKEdpBswGTEXMBUGA1UEAwwO +QXJnb0NEIFRlc3QgQ0GCFAYE19ROoSw/pDCerQyk31Vo99PFMBMGA1UdJQQMMAoG +CCsGAQUFBwMCMAsGA1UdDwQEAwIHgDANBgkqhkiG9w0BAQsFAAOCAQEAA6jUOxdm +/JMgsGj5ef7N4FTWpCD+FHXJY/Cj/0qluNnIQ/oLnNoL/SO5y8CeWttyIZ7FVoEy +FE7V75uXq7iTH3lBsPpmkyiTlVRKiicmiv6B/aVo8pucbGPDlpia6eVtNGnx6sp4 +EOQr4UG/3LbIunbqF2k+z3W4KAMXBg/lmss2J4XXuBOSaRzOcvtxHziiIvqGEyBE +eXefqxHobmWUt+7EOb2JRUxVgJKmg4ODdTwwTtprS3QOp4ZOT3nT0qY41ep9/F+n +czuX78tJCBCWE2RoSNCz61mTnSK6DIMcdKH2YTS2juXhJVoJ7H6wuf0hfmVdOxXX +oKPhT/pOd5A8gw== +-----END CERTIFICATE----- diff --git a/ext/git/testdata/certs/argocd-test-client.key b/ext/git/testdata/certs/argocd-test-client.key new file mode 100644 index 0000000..ce385d4 --- /dev/null +++ b/ext/git/testdata/certs/argocd-test-client.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCWHgTMNRSHmINY +hyGJLECmQCVgatiF/bDC1waNSgzD/yqYd2Dx5nRJsl2L7+OADuNu4FB9E7j+iTqr +pQUjbbpt+D6zdZDIQT1AyBCNYqipS1prVpWUicoo9Hqkdwc6NRWxeQQ6Q3RwUeec +QSmS0/NpZxodSr4NbnK/KXJ9HElUVJMg838AQuGYEleOOaeHb/UTPCW+mwBfVwfI +9Pi4yjb9KTVJcDFmC+elNvsQuob5GJgX2X3gYE/PCIVXjYvgGf6DKImYbyrVheoe +WQwE9Ye2/6KNTGLtaLqafy+slMhyTiT3N1QZ9RRlPmX/e+T5wUKAp4cVKbl4JvEC +8k13uAR5AgMBAAECggEAQNWsOso+GKY9LDIIwOb08RjJS9A5vf0op64Y7VLrGoeN +TRZaL3/Z/65iirrL5hYIEm4dNTgccQqx5Uo7YubUWwSZiAahxmuu2djOlVHkCGI8 +JhnaNrIgNvoIMhoabABbYzAiLEvP8WbegnT+UKTr/z0BYV9ToBdwxbFP+ksKPLo1 +DZYJyKFA+h9UCDR3/woPSOp57Ta/S26BZSqCYeX7PKkzUn8OnfBG16wPfSbMQzM1 +AMwNG0VnQWpkbAhISLRcFUhyQC6kBqu8QEBHYa6SZ0qosGxZhnVo18bhz55q/6s/ +GHf5nKCkN8rBOt5G22tJ22L1s7DLjrRmeyvNogPctQKBgQDFz5ZU/bWAmGFG60lJ +5KtMoGo4Us/8j4sjwMDaoujGO/lsWKejhL4ERfqXm4/yCtDGtq+eOgXN4rOGr/b2 +SW4CCsJYdgiUdCH6A7pmUk9zKT0226Z7YcsBX14pttqni2NTcIvvWKjBDxo9r00f +OaQJbER3s0cwojvBiZs9bDE9zwKBgQDCRsyfiPyp+7NJ2u2Kx9KFIW6j3cOA5K5/ +ixpolsubL1McvTWL6JFcZQ+y7/oQbmX860CFgtKhMOsdOH2AY4UwG5A1BIvdxgwH +E6RJAwa1j/KXS8NjtTGKn7ILPwwlYyFGzCpiaDoqeGIjXD5G5bXMZ+LNMKSdXuw2 +/mOHDVSzNwKBgQCl4VTh5PhF5IL+4+RLsRTtZ0BsBxYfZ4h47PVM43xscHLTpuy9 +tV1bXAuvA2cMxIEbgqt29pVTeB6wffONyToVQEyFvkWnqFOtw16W28OAgT6yODQ+ +F14TwpPGS27FPaCHokPW7PRnIXER9WWpH78tn7sy3gZ/BC00OV8TfR02BQKBgQDB +Fz8vXSbLB/j1cdvgsKRjX4wo4DP8+s0YxTfVNfNd5RZ1HPWIfflj3KpSzcM7644A +aA1z5UfVn9o+67OJFOD+pJaaq08ceEfiDLqfOpvDfzO8/jdP9Xos7nY2SU6YJkOf +qzKBJliRd58KyBa5vnwHkkVQbYVfSEX8jrB7PVuu1wKBgQCLE6GkaJo2rwhMXGe+ +0EzjFwo9wH/zY17xfFzj0rnJ07A16Q3dAFz+BXF90BWQZeWGxOa2NhlStBEAskU4 +7PR5bV0u+G4EnD1TQbrh4jgNswrCCJylK7GjZutB+JCbA3UY3ih8QUlnlhbyoEk9 +xDNMYUwlZfcAqlI/KbmKsEm6/w== +-----END PRIVATE KEY----- diff --git a/ext/git/workaround.go b/ext/git/workaround.go new file mode 100644 index 0000000..5e89acf --- /dev/null +++ b/ext/git/workaround.go @@ -0,0 +1,75 @@ +package git + +import ( + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/transport" + "gopkg.in/src-d/go-git.v4/plumbing/transport/client" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" + "gopkg.in/src-d/go-git.v4/utils/ioutil" +) + +// Below is a workaround for https://github.com/src-d/go-git/issues/1177: the `github.com/src-d/go-git` does not support disable SSL cert verification is a single repo. +// As workaround methods `newUploadPackSession`, `newClient` and `listRemote` were copied from https://github.com/src-d/go-git/blob/master/remote.go and modified to use +// transport with InsecureSkipVerify flag is verification should be disabled. + +func newUploadPackSession(url string, auth transport.AuthMethod, insecure bool, creds Creds) (transport.UploadPackSession, error) { + c, ep, err := newClient(url, insecure, creds) + if err != nil { + return nil, err + } + + return c.NewUploadPackSession(ep, auth) +} + +func newClient(url string, insecure bool, creds Creds) (transport.Transport, *transport.Endpoint, error) { + ep, err := transport.NewEndpoint(url) + if err != nil { + return nil, nil, err + } + + var c transport.Transport + // For HTTPS repositories, we get a custom Transport. Everything else will + // be default. + if IsHTTPSURL(url) { + c = http.NewClient(GetRepoHTTPClient(url, insecure, creds)) + } else { + c, err = client.NewClient(ep) + if err != nil { + return nil, nil, err + } + } + return c, ep, err +} + +func listRemote(r *git.Remote, o *git.ListOptions, insecure bool, creds Creds) (rfs []*plumbing.Reference, err error) { + s, err := newUploadPackSession(r.Config().URLs[0], o.Auth, insecure, creds) + if err != nil { + return nil, err + } + + defer ioutil.CheckClose(s, &err) + + ar, err := s.AdvertisedReferences() + if err != nil { + return nil, err + } + + allRefs, err := ar.AllReferences() + if err != nil { + return nil, err + } + + refs, err := allRefs.IterReferences() + if err != nil { + return nil, err + } + + var resultRefs []*plumbing.Reference + _ = refs.ForEach(func(ref *plumbing.Reference) error { + resultRefs = append(resultRefs, ref) + return nil + }) + + return resultRefs, nil +} |
