Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
67 commits
Select commit Hold shift + click to select a range
c20168c
wip: Probably working right.
yhakbar Feb 21, 2025
a521804
fix: Cleaning things up a bit
yhakbar Feb 24, 2025
cb731e6
fix: Getting rid of unused constant
yhakbar Feb 24, 2025
1a54f04
fix: Fixing remote reference optimization
yhakbar Feb 24, 2025
6d70093
fix: More performance improvements
yhakbar Feb 24, 2025
0b592ea
fix: Performance testing done.
yhakbar Feb 24, 2025
52a9cd1
fix: Tossing some unnecessary comments
yhakbar Feb 24, 2025
4802efe
feat: Attempting implementation
yhakbar Feb 24, 2025
5629b2c
fix: Fixing recursive storage of trees
yhakbar Feb 24, 2025
9264dc3
feat: Upgrade to `go-getter` v2
yhakbar Feb 25, 2025
15d4fa3
feat: Use go-getter for cln protocol
yhakbar Feb 25, 2025
629f98b
fix: Removing debug logging
yhakbar Feb 25, 2025
926fc74
fix: Using an early return
yhakbar Feb 25, 2025
add47f5
fix: Some updates to how scaffolding gets done after catalog
yhakbar Feb 25, 2025
67ce742
fix: Updating experiment name
yhakbar Feb 25, 2025
1f4704f
fix: Rename `clngo` to `cln`
yhakbar Feb 25, 2025
95f3919
fix: Linting
yhakbar Feb 25, 2025
28aad5a
fix: Revert changes to `catalog` command
yhakbar Feb 25, 2025
7ab6f05
fix: Refactoring clone so that it's easier to work with
yhakbar Feb 25, 2025
477e496
fix: Switch to explicit sentinel instead of relying on `.git`
yhakbar Feb 26, 2025
4674e3f
fix: Renaming `repo` to `url`
yhakbar Feb 26, 2025
1549810
fix: Fixing tree linking
yhakbar Feb 26, 2025
0ee6b48
fix: Mostly fixed
yhakbar Feb 26, 2025
23192d4
fix: Tests fixed
yhakbar Feb 26, 2025
46bf743
fix: Fully repaired cln
yhakbar Feb 26, 2025
172901f
feat: Concurrently linking the tree
yhakbar Feb 26, 2025
06c4fe3
feat: Reducing concurrency for linking
yhakbar Feb 27, 2025
09950d8
feat: Persisting select `.git` files
yhakbar Feb 27, 2025
b2650f2
feat: Integrating `cln` into catalog
yhakbar Feb 27, 2025
996d9ad
feat: Renaming cln to CAS
yhakbar Feb 27, 2025
28e4e69
feat: Adjusting store path
yhakbar Feb 27, 2025
ac38312
feat: Paritioning content
yhakbar Feb 27, 2025
acebb90
fix: Refactoring a bit
yhakbar Feb 27, 2025
d69785f
fix: Lock access to the map
yhakbar Feb 27, 2025
557290b
fix: Improving integration into catalog
yhakbar Feb 27, 2025
2196a3a
fix: Linting
yhakbar Feb 27, 2025
18b7990
feat: Adding experiment docs
yhakbar Feb 27, 2025
5f2ab42
feat: Adding CAS feature docs
yhakbar Feb 27, 2025
8a66be5
fix: Filling in placeholder link
yhakbar Feb 27, 2025
d0f60f7
fix: Addressing CodeRabbit feedback
yhakbar Feb 27, 2025
27daacb
fix: Use parent logger
yhakbar Feb 27, 2025
21996f6
fix: Linting errors
yhakbar Feb 27, 2025
8f9b5ce
fix: Working on optimization
yhakbar Mar 4, 2025
b3b4d6f
fix: Tests working
yhakbar Mar 4, 2025
5e692a5
fix: Cleanup
yhakbar Mar 4, 2025
97d7859
fix: Adding tmp timestamp check
yhakbar Mar 4, 2025
60e7b2b
fix: Remove unused constants
yhakbar Mar 4, 2025
4c53664
feat: Adding `LinkTree` test
yhakbar Mar 4, 2025
9b1a4fc
fix: Update implementation to test table
yhakbar Mar 4, 2025
456c68c
fix: Linting
yhakbar Mar 4, 2025
53d2566
fix: Deleting some dead code
yhakbar Mar 4, 2025
12fe862
fix: Fixing Windows test of hard linked tree
yhakbar Mar 4, 2025
adf258f
fix: Fixing unnecessary imports
yhakbar Mar 4, 2025
cb71191
feat: Bumping Catalog to use go-getter v2
yhakbar Mar 4, 2025
58f3ea3
wip: Adding go-getter
yhakbar Mar 4, 2025
3844890
fix: Adjusting abstraction so that clone gets options
yhakbar Mar 4, 2025
baa76b0
fix: Refactoring
yhakbar Mar 4, 2025
1e0805d
fix: Adding getter
yhakbar Mar 4, 2025
ebe538c
fix: Use go-getter for clones
yhakbar Mar 4, 2025
87617ad
fix: Updating docs for updated CAS implementation
yhakbar Mar 4, 2025
edd85b1
fix: Linting
yhakbar Mar 5, 2025
428d562
fix: Getting rid of command cache
yhakbar Mar 5, 2025
46b8000
fix: Fixing comment
yhakbar Mar 5, 2025
c6efc57
fix: Addressing CodeRabbit feedback
yhakbar Mar 5, 2025
f50e994
fix: Removing unnecessary lock
yhakbar Mar 5, 2025
49291b1
feat: Plumbing ctx
yhakbar Mar 5, 2025
5086658
feat: Addressing review feedback
yhakbar Mar 10, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat: Plumbing ctx
  • Loading branch information
yhakbar committed Mar 10, 2025
commit 49291b1bb30aec8a61590129133ce90b3be77325
13 changes: 8 additions & 5 deletions internal/cas/benchmark_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,15 +129,18 @@ func BenchmarkGitOperations(b *testing.B) {
// Setup a git repository for testing
repoDir := b.TempDir()
git := cas.NewGitRunner().WithWorkDir(repoDir)
if err := git.Clone("https://github.com/gruntwork-io/terragrunt.git", false, 1, "main"); err != nil {

ctx := context.Background()

if err := git.Clone(ctx, "https://github.com/gruntwork-io/terragrunt.git", false, 1, "main"); err != nil {
b.Fatal(err)
}

b.Run("ls-remote", func(b *testing.B) {
git := cas.NewGitRunner() // No workDir needed for ls-remote
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := git.LsRemote("https://github.com/gruntwork-io/terragrunt.git", "HEAD")
_, err := git.LsRemote(ctx, "https://github.com/gruntwork-io/terragrunt.git", "HEAD")
if err != nil {
b.Fatal(err)
}
Expand All @@ -147,7 +150,7 @@ func BenchmarkGitOperations(b *testing.B) {
b.Run("ls-tree", func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := git.LsTree("HEAD", ".")
_, err := git.LsTree(ctx, "HEAD", ".")
if err != nil {
b.Fatal(err)
}
Expand All @@ -156,7 +159,7 @@ func BenchmarkGitOperations(b *testing.B) {

b.Run("cat-file", func(b *testing.B) {
// First get a valid hash
tree, err := git.LsTree("HEAD", ".")
tree, err := git.LsTree(ctx, "HEAD", ".")
if err != nil {
b.Fatal(err)
}
Expand All @@ -178,7 +181,7 @@ func BenchmarkGitOperations(b *testing.B) {

b.ResetTimer()
for i := 0; i < b.N; i++ {
err := git.CatFile(hash, tmp)
err := git.CatFile(ctx, hash, tmp)
if err != nil {
b.Fatal(err)
}
Expand Down
38 changes: 19 additions & 19 deletions internal/cas/cas.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,13 +95,13 @@ func (c *CAS) Clone(ctx context.Context, l *log.Logger, opts CloneOptions, url s
// Set the working directory for git operations
c.git.SetWorkDir(tempDir)

hash, err := c.resolveReference(url, opts.Branch)
hash, err := c.resolveReference(ctx, url, opts.Branch)
if err != nil {
return err
}

if c.store.NeedsWrite(hash, c.cloneStart) {
if err := c.cloneAndStoreContent(l, opts, url, hash); err != nil {
if err := c.cloneAndStoreContent(ctx, l, opts, url, hash); err != nil {
return err
}
}
Expand Down Expand Up @@ -134,8 +134,8 @@ func (c *CAS) prepareTargetDirectory(dir, url string) string {
return filepath.Clean(targetDir)
}

func (c *CAS) resolveReference(url, branch string) (string, error) {
results, err := c.git.LsRemote(url, branch)
func (c *CAS) resolveReference(ctx context.Context, url, branch string) (string, error) {
results, err := c.git.LsRemote(ctx, url, branch)
if err != nil {
return "", err
}
Expand All @@ -151,16 +151,16 @@ func (c *CAS) resolveReference(url, branch string) (string, error) {
return results[0].Hash, nil
}

func (c *CAS) cloneAndStoreContent(l *log.Logger, opts CloneOptions, url string, hash string) error {
if err := c.git.Clone(url, true, 1, opts.Branch); err != nil {
func (c *CAS) cloneAndStoreContent(ctx context.Context, l *log.Logger, opts CloneOptions, url string, hash string) error {
if err := c.git.Clone(ctx, url, true, 1, opts.Branch); err != nil {
return err
}

return c.storeRootTree(l, hash, opts)
return c.storeRootTree(ctx, l, hash, opts)
}

func (c *CAS) storeRootTree(l *log.Logger, hash string, opts CloneOptions) error {
if err := c.storeTree(l, hash, ""); err != nil {
func (c *CAS) storeRootTree(ctx context.Context, l *log.Logger, hash string, opts CloneOptions) error {
if err := c.storeTree(ctx, l, hash, ""); err != nil {
return err
}

Expand Down Expand Up @@ -207,12 +207,12 @@ func (c *CAS) storeRootTree(l *log.Logger, hash string, opts CloneOptions) error
return content.Store(l, hash, data)
}

func (c *CAS) storeTree(l *log.Logger, hash, prefix string) error {
func (c *CAS) storeTree(ctx context.Context, l *log.Logger, hash, prefix string) error {
if !c.store.NeedsWrite(hash, c.cloneStart) {
return nil
}

tree, err := c.git.LsTree(hash, ".")
tree, err := c.git.LsTree(ctx, hash, ".")
if err != nil {
return err
}
Expand Down Expand Up @@ -245,7 +245,7 @@ func (c *CAS) storeTree(l *log.Logger, hash, prefix string) error {
go func() {
defer wg.Done()

if err := c.storeBlobs(blobs); err != nil {
if err := c.storeBlobs(ctx, blobs); err != nil {
ch <- err

return
Expand All @@ -259,7 +259,7 @@ func (c *CAS) storeTree(l *log.Logger, hash, prefix string) error {
go func() {
defer wg.Done()

if err := c.storeTrees(l, trees, prefix); err != nil {
if err := c.storeTrees(ctx, l, trees, prefix); err != nil {
ch <- err

return
Expand Down Expand Up @@ -294,7 +294,7 @@ func (c *CAS) storeTree(l *log.Logger, hash, prefix string) error {
}

// storeBlobs concurrently stores blobs in the CAS
func (c *CAS) storeBlobs(entries []TreeEntry) error {
func (c *CAS) storeBlobs(ctx context.Context, entries []TreeEntry) error {
ch := make(chan error, len(entries))

var wg sync.WaitGroup
Expand All @@ -309,7 +309,7 @@ func (c *CAS) storeBlobs(entries []TreeEntry) error {
go func(hash string) {
defer wg.Done()

if err := c.ensureBlob(hash); err != nil {
if err := c.ensureBlob(ctx, hash); err != nil {
ch <- err

return
Expand Down Expand Up @@ -339,7 +339,7 @@ func (c *CAS) storeBlobs(entries []TreeEntry) error {
}

// storeTrees concurrently stores trees in the CAS
func (c *CAS) storeTrees(l *log.Logger, entries []TreeEntry, prefix string) error {
func (c *CAS) storeTrees(ctx context.Context, l *log.Logger, entries []TreeEntry, prefix string) error {
ch := make(chan error, len(entries))

var wg sync.WaitGroup
Expand All @@ -354,7 +354,7 @@ func (c *CAS) storeTrees(l *log.Logger, entries []TreeEntry, prefix string) erro
go func(hash string) {
defer wg.Done()

if err := c.storeTree(l, hash, prefix); err != nil {
if err := c.storeTree(ctx, l, hash, prefix); err != nil {
ch <- err

return
Expand Down Expand Up @@ -387,7 +387,7 @@ func (c *CAS) storeTrees(l *log.Logger, entries []TreeEntry, prefix string) erro
// It doesn't use the standard content.Store method because
// we want to take advantage of the ability to write to the
// entry using `git cat-file`.
func (c *CAS) ensureBlob(hash string) (err error) {
func (c *CAS) ensureBlob(ctx context.Context, hash string) (err error) {
c.store.mapLock.Lock()

if _, ok := c.store.locks[hash]; !ok {
Expand Down Expand Up @@ -420,7 +420,7 @@ func (c *CAS) ensureBlob(hash string) (err error) {
}
}()

err = c.git.CatFile(hash, tmpHandle)
err = c.git.CatFile(ctx, hash, tmpHandle)
if err != nil {
return err
}
Expand Down
21 changes: 11 additions & 10 deletions internal/cas/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package cas

import (
"bytes"
"context"
"io"
"os"
"os/exec"
Expand Down Expand Up @@ -52,14 +53,14 @@ type LsRemoteResult struct {

// LsRemote runs git ls-remote for a specific reference.
// If ref is empty, we check HEAD instead.
func (g *GitRunner) LsRemote(repo, ref string) ([]LsRemoteResult, error) {
func (g *GitRunner) LsRemote(ctx context.Context, repo, ref string) ([]LsRemoteResult, error) {
if ref == "" {
ref = "HEAD"
}

args := []string{repo, ref}

cmd := g.prepareCommand("ls-remote", args...)
cmd := g.prepareCommand(ctx, "ls-remote", args...)

var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
Expand Down Expand Up @@ -103,7 +104,7 @@ func (g *GitRunner) LsRemote(repo, ref string) ([]LsRemoteResult, error) {
}

// Clone performs a git clone operation
func (g *GitRunner) Clone(repo string, bare bool, depth int, branch string) error {
func (g *GitRunner) Clone(ctx context.Context, repo string, bare bool, depth int, branch string) error {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was wondering why CloneOptions is not passed as pointer?

Should it be added checks to validate repo/branch?

if err := g.RequiresWorkDir(); err != nil {
return err
}
Expand All @@ -124,7 +125,7 @@ func (g *GitRunner) Clone(repo string, bare bool, depth int, branch string) erro

args = append(args, repo, g.WorkDir)

cmd := g.prepareCommand("clone", args...)
cmd := g.prepareCommand(ctx, "clone", args...)

var stderr bytes.Buffer
cmd.Stderr = &stderr
Expand Down Expand Up @@ -175,12 +176,12 @@ func GetRepoName(repo string) string {
}

// LsTree runs git ls-tree and returns the parsed tree
func (g *GitRunner) LsTree(reference, path string) (*Tree, error) {
func (g *GitRunner) LsTree(ctx context.Context, reference, path string) (*Tree, error) {
if err := g.RequiresWorkDir(); err != nil {
return nil, err
}

cmd := g.prepareCommand("ls-tree", reference)
cmd := g.prepareCommand(ctx, "ls-tree", reference)
cmd.Dir = g.WorkDir

var stdout, stderr bytes.Buffer
Expand All @@ -200,14 +201,14 @@ func (g *GitRunner) LsTree(reference, path string) (*Tree, error) {

// CatFile writes the contents of a git object
// to a given writer.
func (g *GitRunner) CatFile(hash string, out io.Writer) error {
func (g *GitRunner) CatFile(ctx context.Context, hash string, out io.Writer) error {
if err := g.RequiresWorkDir(); err != nil {
return err
}

var stderr bytes.Buffer

cmd := g.prepareCommand("cat-file", "-p", hash)
cmd := g.prepareCommand(ctx, "cat-file", "-p", hash)
cmd.Dir = g.WorkDir
cmd.Stdout = out
cmd.Stderr = &stderr
Expand All @@ -231,6 +232,6 @@ func (g *GitRunner) SetWorkDir(dir string) {
g.WorkDir = dir
}

func (g *GitRunner) prepareCommand(name string, args ...string) *exec.Cmd {
return exec.Command("git", append([]string{name}, args...)...)
func (g *GitRunner) prepareCommand(ctx context.Context, name string, args ...string) *exec.Cmd {
return exec.CommandContext(ctx, "git", append([]string{name}, args...)...)
}
31 changes: 19 additions & 12 deletions internal/cas/git_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cas_test

import (
"context"
"os"
"path/filepath"
"testing"
Expand All @@ -14,9 +15,11 @@ func TestGitRunner_LsRemote(t *testing.T) {
t.Parallel()
runner := cas.NewGitRunner()

ctx := context.Background()

t.Run("valid repository", func(t *testing.T) {
t.Parallel()
results, err := runner.LsRemote("https://github.com/gruntwork-io/terragrunt.git", "HEAD")
results, err := runner.LsRemote(ctx, "https://github.com/gruntwork-io/terragrunt.git", "HEAD")
require.NoError(t, err)
require.NotEmpty(t, results)
assert.Regexp(t, "^[0-9a-f]{40}$", results[0].Hash)
Expand All @@ -25,7 +28,7 @@ func TestGitRunner_LsRemote(t *testing.T) {

t.Run("invalid repository", func(t *testing.T) {
t.Parallel()
_, err := runner.LsRemote("https://github.com/nonexistent/repo.git", "HEAD")
_, err := runner.LsRemote(ctx, "https://github.com/nonexistent/repo.git", "HEAD")
require.Error(t, err)
var wrappedErr *cas.WrappedError
require.ErrorAs(t, err, &wrappedErr)
Expand All @@ -34,7 +37,7 @@ func TestGitRunner_LsRemote(t *testing.T) {

t.Run("nonexistent reference", func(t *testing.T) {
t.Parallel()
_, err := runner.LsRemote("https://github.com/gruntwork-io/terragrunt.git", "nonexistent-branch")
_, err := runner.LsRemote(ctx, "https://github.com/gruntwork-io/terragrunt.git", "nonexistent-branch")
require.Error(t, err)
var wrappedErr *cas.WrappedError
require.ErrorAs(t, err, &wrappedErr)
Expand All @@ -45,11 +48,13 @@ func TestGitRunner_LsRemote(t *testing.T) {
func TestGitRunner_Clone(t *testing.T) {
t.Parallel()

ctx := context.Background()

t.Run("shallow clone", func(t *testing.T) {
t.Parallel()
cloneDir := t.TempDir()
runner := cas.NewGitRunner().WithWorkDir(cloneDir)
err := runner.Clone("https://github.com/gruntwork-io/terragrunt.git", true, 1, "main")
err := runner.Clone(ctx, "https://github.com/gruntwork-io/terragrunt.git", true, 1, "main")
require.NoError(t, err)

// Verify it's a git repository
Expand All @@ -60,7 +65,7 @@ func TestGitRunner_Clone(t *testing.T) {
t.Run("clone without workdir fails", func(t *testing.T) {
t.Parallel()
runner := cas.NewGitRunner()
err := runner.Clone("https://github.com/gruntwork-io/terragrunt.git", true, 1, "main")
err := runner.Clone(ctx, "https://github.com/gruntwork-io/terragrunt.git", true, 1, "main")
require.Error(t, err)
var wrappedErr *cas.WrappedError
require.ErrorAs(t, err, &wrappedErr)
Expand All @@ -71,7 +76,7 @@ func TestGitRunner_Clone(t *testing.T) {
t.Parallel()
cloneDir := t.TempDir()
runner := cas.NewGitRunner().WithWorkDir(cloneDir)
err := runner.Clone("https://github.com/gruntwork-io/terragrunt-fake.git", false, 1, "")
err := runner.Clone(ctx, "https://github.com/gruntwork-io/terragrunt-fake.git", false, 1, "")
require.Error(t, err)
var wrappedErr *cas.WrappedError
require.ErrorAs(t, err, &wrappedErr)
Expand Down Expand Up @@ -133,17 +138,19 @@ func TestGetRepoName(t *testing.T) {
func TestGitRunner_LsTree(t *testing.T) {
t.Parallel()

ctx := context.Background()

t.Run("valid repository", func(t *testing.T) {
t.Parallel()
cloneDir := t.TempDir()
runner := cas.NewGitRunner().WithWorkDir(cloneDir)

// First clone a repository
err := runner.Clone("https://github.com/gruntwork-io/terragrunt.git", true, 1, "main")
err := runner.Clone(ctx, "https://github.com/gruntwork-io/terragrunt.git", true, 1, "main")
require.NoError(t, err)

// Then try to ls-tree HEAD
tree, err := runner.LsTree("HEAD", ".")
tree, err := runner.LsTree(ctx, "HEAD", ".")
require.NoError(t, err)
require.NotEmpty(t, tree.Entries())

Expand All @@ -164,7 +171,7 @@ func TestGitRunner_LsTree(t *testing.T) {
t.Run("ls-tree without workdir fails", func(t *testing.T) {
t.Parallel()
runner := cas.NewGitRunner()
_, err := runner.LsTree("HEAD", ".")
_, err := runner.LsTree(ctx, "HEAD", ".")
require.Error(t, err)
var wrappedErr *cas.WrappedError
require.ErrorAs(t, err, &wrappedErr)
Expand All @@ -177,11 +184,11 @@ func TestGitRunner_LsTree(t *testing.T) {
runner := cas.NewGitRunner().WithWorkDir(cloneDir)

// First clone a repository
err := runner.Clone("https://github.com/gruntwork-io/terragrunt.git", true, 1, "main")
err := runner.Clone(ctx, "https://github.com/gruntwork-io/terragrunt.git", true, 1, "main")
require.NoError(t, err)

// Try to ls-tree an invalid reference
_, err = runner.LsTree("nonexistent", ".")
_, err = runner.LsTree(ctx, "nonexistent", ".")
require.Error(t, err)
var wrappedErr *cas.WrappedError
require.ErrorAs(t, err, &wrappedErr)
Expand All @@ -193,7 +200,7 @@ func TestGitRunner_LsTree(t *testing.T) {
runner := cas.NewGitRunner().WithWorkDir(t.TempDir())

// Try to ls-tree in an empty directory
_, err := runner.LsTree("HEAD", ".")
_, err := runner.LsTree(ctx, "HEAD", ".")
require.Error(t, err)
var wrappedErr *cas.WrappedError
require.ErrorAs(t, err, &wrappedErr)
Expand Down
Loading