From 859900ef39bbaa54d18298615fca843e0488b106 Mon Sep 17 00:00:00 2001 From: jnovick Date: Wed, 14 Aug 2024 04:22:00 +0300 Subject: feat: Use shallow clone to speed up performance (#830) Signed-off-by: Joshua Novick --- ext/git/client.go | 1 + ext/git/git_test.go | 19 +++++++++++++++++++ ext/git/mocks/Client.go | 18 ++++++++++++++++++ ext/git/writer.go | 35 +++++++++++++++++++++++++++++++++++ pkg/argocd/git.go | 8 ++++---- pkg/argocd/update_test.go | 10 +++++----- 6 files changed, 82 insertions(+), 9 deletions(-) diff --git a/ext/git/client.go b/ext/git/client.go index ac9dc10..199716b 100644 --- a/ext/git/client.go +++ b/ext/git/client.go @@ -65,6 +65,7 @@ type Client interface { Root() string Init() error Fetch(revision string) error + ShallowFetch(revision string, depth int) error Submodule() error Checkout(revision string, submoduleEnabled bool) error LsRefs() (*Refs, error) diff --git a/ext/git/git_test.go b/ext/git/git_test.go index 0eebe35..508ecf8 100644 --- a/ext/git/git_test.go +++ b/ext/git/git_test.go @@ -322,6 +322,25 @@ func TestVerifyCommitSignature(t *testing.T) { } } +func TestVerifyShallowFetchCheckout(t *testing.T) { + p := t.TempDir() + + client, err := NewClientExt("https://github.com/argoproj/argo-cd.git", p, NopCreds{}, false, false, "") + assert.NoError(t, err) + + err = client.Init() + assert.NoError(t, err) + + err = client.ShallowFetch("HEAD", 1) + assert.NoError(t, err) + + commitSHA, err := client.LsRemote("HEAD") + assert.NoError(t, err) + + err = client.Checkout(commitSHA, true) + assert.NoError(t, err) +} + func TestNewFactory(t *testing.T) { addBinDirToPath := path.NewBinDirToPath() defer addBinDirToPath.Close() diff --git a/ext/git/mocks/Client.go b/ext/git/mocks/Client.go index 87bd3b4..ffe2b91 100644 --- a/ext/git/mocks/Client.go +++ b/ext/git/mocks/Client.go @@ -192,6 +192,24 @@ func (_m *Client) Fetch(revision string) error { return r0 } +// Fetch provides a mock function with given fields: revision +func (_m *Client) ShallowFetch(revision string, depth int) error { + ret := _m.Called(revision) + + if len(ret) == 0 { + panic("no return value specified for Fetch") + } + + var r0 error + if rf, ok := ret.Get(0).(func(string, int) error); ok { + r0 = rf(revision, depth) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Init provides a mock function with given fields: func (_m *Client) Init() error { ret := _m.Called() diff --git a/ext/git/writer.go b/ext/git/writer.go index 8e6f0f2..d2d2f75 100644 --- a/ext/git/writer.go +++ b/ext/git/writer.go @@ -3,6 +3,7 @@ package git import ( "fmt" "os/exec" + "strconv" "strings" "github.com/argoproj-labs/argocd-image-updater/pkg/log" @@ -159,3 +160,37 @@ func (m *nativeGitClient) runCredentialedCmdWithOutput(args ...string) (string, cmd.Env = append(cmd.Env, environ...) return m.runCmdOutput(cmd, runOpts{}) } + +func (m *nativeGitClient) shallowFetch(revision string, depth int) error { + var err error + if revision != "" { + err = m.runCredentialedCmd("fetch", "origin", revision, "--force", "--prune", "--depth", strconv.Itoa(depth)) + } else { + err = m.runCredentialedCmd("fetch", "origin", "--force", "--prune", "--depth", strconv.Itoa(depth)) + } + return err +} + +// Fetch fetches latest updates from origin +func (m *nativeGitClient) ShallowFetch(revision string, depth int) error { + if m.OnFetch != nil { + done := m.OnFetch(m.repoURL) + defer done() + } + + err := m.shallowFetch(revision, depth) + + // When we have LFS support enabled, check for large files and fetch them too. + // No shallow fetch is possible here + if err == nil && m.IsLFSEnabled() { + largeFiles, err := m.LsLargeFiles() + if err == nil && len(largeFiles) > 0 { + err = m.runCredentialedCmd("lfs", "fetch", "--all") + if err != nil { + return err + } + } + } + + return err +} diff --git a/pkg/argocd/git.go b/pkg/argocd/git.go index f984790..b2cd899 100644 --- a/pkg/argocd/git.go +++ b/pkg/argocd/git.go @@ -157,10 +157,6 @@ func commitChangesGit(app *v1alpha1.Application, wbc *WriteBackConfig, changeLis if err != nil { return err } - err = gitC.Fetch("") - if err != nil { - return err - } // Set username and e-mail address used to identify the commiter if wbc.GitCommitUser != "" && wbc.GitCommitEmail != "" { @@ -186,6 +182,10 @@ func commitChangesGit(app *v1alpha1.Application, wbc *WriteBackConfig, changeLis return err } } + err = gitC.ShallowFetch(checkOutBranch, 1) + if err != nil { + return err + } // The push branch is by default the same as the checkout branch, unless // specified after a : separator git-branch annotation, in which case a diff --git a/pkg/argocd/update_test.go b/pkg/argocd/update_test.go index 7896a99..4d989e8 100644 --- a/pkg/argocd/update_test.go +++ b/pkg/argocd/update_test.go @@ -3009,7 +3009,7 @@ replacements: [] app := app.DeepCopy() gitMock := &gitmock.Client{} gitMock.On("Init").Return(nil) - gitMock.On("Fetch", mock.Anything).Return(nil) + gitMock.On("ShallowFetch", mock.Anything, mock.Anything).Return(nil) gitMock.On("Checkout", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { args.Assert(t, "mydefaultbranch", false) }).Return(nil) @@ -3035,7 +3035,7 @@ replacements: [] t.Run("Cannot init", func(t *testing.T) { gitMock := &gitmock.Client{} gitMock.On("Init").Return(fmt.Errorf("cannot init")) - gitMock.On("Fetch", mock.Anything).Return(nil) + gitMock.On("ShallowFetch", mock.Anything, mock.Anything).Return(nil) gitMock.On("Checkout", mock.Anything, mock.Anything).Return(nil) gitMock.On("Commit", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) gitMock.On("Push", mock.Anything, mock.Anything, mock.Anything).Return(nil) @@ -3050,7 +3050,7 @@ replacements: [] t.Run("Cannot fetch", func(t *testing.T) { gitMock := &gitmock.Client{} gitMock.On("Init").Return(nil) - gitMock.On("Fetch", mock.Anything).Return(fmt.Errorf("cannot fetch")) + gitMock.On("ShallowFetch", mock.Anything, mock.Anything).Return(fmt.Errorf("cannot fetch")) gitMock.On("Checkout", mock.Anything, mock.Anything).Return(nil) gitMock.On("Commit", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) gitMock.On("Push", mock.Anything, mock.Anything, mock.Anything).Return(nil) @@ -3064,7 +3064,7 @@ replacements: [] t.Run("Cannot checkout", func(t *testing.T) { gitMock := &gitmock.Client{} gitMock.On("Init").Return(nil) - gitMock.On("Fetch", mock.Anything).Return(nil) + gitMock.On("ShallowFetch", mock.Anything, mock.Anything).Return(nil) gitMock.On("Checkout", mock.Anything, mock.Anything).Return(fmt.Errorf("cannot checkout")) gitMock.On("Commit", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) gitMock.On("Push", mock.Anything, mock.Anything, mock.Anything).Return(nil) @@ -3180,7 +3180,7 @@ func mockGit(t *testing.T) (gitMock *gitmock.Client, dir string, cleanup func()) gitMock = &gitmock.Client{} gitMock.On("Root").Return(dir) gitMock.On("Init").Return(nil) - gitMock.On("Fetch", mock.Anything).Return(nil) + gitMock.On("ShallowFetch", mock.Anything, mock.Anything).Return(nil) return gitMock, dir, func() { _ = os.RemoveAll(dir) } -- cgit v1.2.3