feat: Adding cas support to catalog

cli/app_test.go

@@ -533,8 +533,6 @@ func (err argMissingValueError) Error() string {
 }
 
 func TestAutocomplete(t *testing.T) { //nolint:paralleltest
-	defer os.Unsetenv("COMP_LINE")
-
 	testCases := []struct {
 		compLine          string
 		expectedCompletes []string
@@ -558,7 +556,7 @@ func TestAutocomplete(t *testing.T) { //nolint:paralleltest
 	}
 
 	for _, testCase := range testCases {
-		os.Setenv("COMP_LINE", "terragrunt "+testCase.compLine)
+		t.Setenv("COMP_LINE", "terragrunt "+testCase.compLine)
 
 		output := &bytes.Buffer{}
 		opts := options.NewTerragruntOptionsWithWriters(output, os.Stderr)

cli/commands/catalog/action.go

@@ -38,11 +38,12 @@ func Run(ctx context.Context, opts *options.TerragruntOptions, repoURL string) e
 	var modules module.Modules
 
 	walkWithSymlinks := opts.Experiments.Evaluate(experiment.Symlinks)
+	allowCAS := opts.Experiments.Evaluate(experiment.CAS)
 
 	for _, repoURL := range repoURLs {
-		tempDir := filepath.Join(os.TempDir(), fmt.Sprintf(tempDirFormat, util.EncodeBase64Sha1(repoURL)))
+		path := filepath.Join(os.TempDir(), fmt.Sprintf(tempDirFormat, util.EncodeBase64Sha1(repoURL)))
 
-		repo, err := module.NewRepo(ctx, opts.Logger, repoURL, tempDir, walkWithSymlinks)
+		repo, err := module.NewRepo(ctx, opts.Logger, repoURL, path, walkWithSymlinks, allowCAS)
 		if err != nil {
 			return err
 		}

cli/commands/catalog/module/module.go

@@ -32,7 +32,7 @@ type Module struct {
 	url       string
 }
 
-// NewModule returns a module instance if the given `moduleDir` path contains a Terraform module, otherwise returns nil.
+// NewModule returns a module instance if the given `moduleDir` path contains an OpenTofu/Terraform module, otherwise returns nil.
 func NewModule(repo *Repo, moduleDir string) (*Module, error) {
 	module := &Module{
 		Repo:      repo,

cli/commands/catalog/module/repo.go

@@ -3,7 +3,6 @@ package module
 import (
 	"context"
 	"fmt"
-	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -13,10 +12,11 @@ import (
 
 	"github.com/gitsight/go-vcsurl"
 	"github.com/gruntwork-io/go-commons/files"
+	"github.com/gruntwork-io/terragrunt/internal/cas"
 	"github.com/gruntwork-io/terragrunt/internal/errors"
 	"github.com/gruntwork-io/terragrunt/pkg/log"
 	"github.com/gruntwork-io/terragrunt/tf"
-	"github.com/hashicorp/go-getter"
+	"github.com/hashicorp/go-getter/v2"
 	"gopkg.in/ini.v1"
 )
 
@@ -27,13 +27,17 @@ const (
 	azuredevHost          = "dev.azure.com"
 	bitbucketHost         = "bitbucket.org"
 	gitlabSelfHostedRegex = `^(gitlab\.(.+))$`
+
+	cloneCompleteSentinel = ".catalog-clone-complete"
 )
 
 var (
 	gitHeadBranchNameReg    = regexp.MustCompile(`^.*?([^/]+)$`)
-	repoNameFromCloneURLReg = regexp.MustCompile(`(?i)^.*?([-a-z_.]+)[^/]*?(?:\.git)?$`)
+	repoNameFromCloneURLReg = regexp.MustCompile(`(?i)^.*?([-a-z0-9_.]+)[^/]*?(?:\.git)?$`)
 
 	modulesPaths = []string{"modules"}
+
+	includedGitFiles = []string{"HEAD", "config"}
 )
 
 type Repo struct {
@@ -46,17 +50,19 @@ type Repo struct {
 	BranchName string
 
 	walkWithSymlinks bool
+	allowCAS         bool
 }
 
-func NewRepo(ctx context.Context, logger log.Logger, cloneURL, tempDir string, walkWithSymlinks bool) (*Repo, error) {
+func NewRepo(ctx context.Context, l log.Logger, cloneURL, path string, walkWithSymlinks bool, allowCAS bool) (*Repo, error) {
 	repo := &Repo{
-		logger:           logger,
+		logger:           l,
 		cloneURL:         cloneURL,
-		path:             tempDir,
+		path:             path,
 		walkWithSymlinks: walkWithSymlinks,
+		allowCAS:         allowCAS,
 	}
 
-	if err := repo.clone(ctx); err != nil {
+	if err := repo.clone(ctx, l); err != nil {
 		return nil, err
 	}
 
@@ -163,69 +169,164 @@ func (repo *Repo) ModuleURL(moduleDir string) (string, error) {
 	return "", errors.Errorf("hosting: %q is not supported yet", remote.Host)
 }
 
-// clone clones the repository to a temporary directory if the repoPath is URL
-func (repo *Repo) clone(ctx context.Context) error {
+type CloneOptions struct {
+	SourceURL  string
+	TargetPath string
+	Context    context.Context
+	Logger     log.Logger
+}
+
+func (repo *Repo) clone(ctx context.Context, l log.Logger) error {
+	cloneURL, err := repo.resolveCloneURL()
+	if err != nil {
+		return err
+	}
+
+	// Handle local directory case
+	if files.IsDir(cloneURL) {
+		return repo.handleLocalDir(cloneURL)
+	}
+
+	// Prepare clone options
+	opts := CloneOptions{
+		SourceURL:  cloneURL,
+		TargetPath: repo.path,
+		Context:    ctx,
+		Logger:     repo.logger,
+	}
+
+	if err := repo.prepareCloneDirectory(); err != nil {
+		return err
+	}
+
+	if repo.cloneCompleted() {
+		repo.logger.Debugf("The repo dir exists and %q exists. Skipping cloning.", cloneCompleteSentinel)
+
+		return nil
+	}
+
+	return repo.performClone(ctx, l, &opts)
+}
+
+func (repo *Repo) resolveCloneURL() (string, error) {
 	if repo.cloneURL == "" {
 		currentDir, err := os.Getwd()
 		if err != nil {
-			return errors.New(err)
+			return "", errors.New(err)
 		}
 
-		repo.cloneURL = currentDir
+		return currentDir, nil
 	}
 
-	if repoPath := repo.cloneURL; files.IsDir(repoPath) {
-		if !filepath.IsAbs(repoPath) {
-			absRepoPath, err := filepath.Abs(repoPath)
-			if err != nil {
-				return errors.New(err)
-			}
+	return repo.cloneURL, nil
+}
 
-			repo.logger.Debugf("Converting relative path %q to absolute %q", repoPath, absRepoPath)
+func (repo *Repo) handleLocalDir(repoPath string) error {
+	if !filepath.IsAbs(repoPath) {
+		absRepoPath, err := filepath.Abs(repoPath)
+		if err != nil {
+			return errors.New(err)
 		}
 
-		repo.path = repoPath
+		repo.logger.Debugf("Converting relative path %q to absolute %q", repoPath, absRepoPath)
+		repo.path = absRepoPath
 
 		return nil
 	}
 
+	repo.path = repoPath
+
+	return nil
+}
+
+func (repo *Repo) prepareCloneDirectory() error {
 	if err := os.MkdirAll(repo.path, os.ModePerm); err != nil {
 		return errors.New(err)
 	}
 
+	repoName := repo.extractRepoName()
+	repo.path = filepath.Join(repo.path, repoName)
+
+	// Clean up incomplete clones
+	if repo.shouldCleanupIncompleteClone() {
+		repo.logger.Debugf("The repo dir exists but %q does not. Removing the repo dir for cloning from the remote source.", cloneCompleteSentinel)
+
+		if err := os.RemoveAll(repo.path); err != nil {
+			return errors.New(err)
+		}
+	}
+
+	return nil
+}
+
+func (repo *Repo) extractRepoName() string {
 	repoName := "temp"
 	if match := repoNameFromCloneURLReg.FindStringSubmatch(repo.cloneURL); len(match) > 0 && match[1] != "" {
 		repoName = match[1]
 	}
 
-	repo.path = filepath.Join(repo.path, repoName)
+	return repoName
+}
 
-	// Since we are cloning the repository into a temporary directory, some operating systems such as MacOS have a service for deleting files that have not been accessed for a long time.
-	// For example, in MacOS the service is responsible for deleting unused files deletes only files while leaving the directory structure is untouched, which in turn misleads `go-getter`, which thinks that the repository exists but cannot update it due to the lack of files. In such cases, we simply delete the temporary directory in order to clone the one again.
-	// See https://github.com/gruntwork-io/terragrunt/pull/2888
-	if files.FileExists(repo.path) && !files.FileExists(repo.gitHeadfile()) {
-		repo.logger.Debugf("The repo dir exists but git file %q does not. Removing the repo dir for cloning from the remote source.", repo.gitHeadfile())
+func (repo *Repo) shouldCleanupIncompleteClone() bool {
+	return files.FileExists(repo.path) && !repo.cloneCompleted()
+}
 
-		if err := os.RemoveAll(repo.path); err != nil {
-			return errors.New(err)
+func (repo *Repo) cloneCompleted() bool {
+	return files.FileExists(filepath.Join(repo.path, cloneCompleteSentinel))
+}
+
+func (repo *Repo) performClone(ctx context.Context, l log.Logger, opts *CloneOptions) error {
+	client := getter.DefaultClient
+
+	if repo.allowCAS {
+		c, err := cas.New(cas.Options{})
+		if err != nil {
+			return err
+		}
+
+		cloneOpts := cas.CloneOptions{
+			Dir:              repo.path,
+			IncludedGitFiles: includedGitFiles,
 		}
+
+		client.Getters = append([]getter.Getter{cas.NewCASGetter(&l, c, &cloneOpts)}, client.Getters...)
 	}
 
-	sourceURL, err := tf.ToSourceURL(repo.cloneURL, "")
+	sourceURL, err := tf.ToSourceURL(opts.SourceURL, "")
 	if err != nil {
 		return err
 	}
 
 	repo.cloneURL = sourceURL.String()
+	opts.Logger.Infof("Cloning repository %q to temporary directory %q", repo.cloneURL, repo.path)
 
-	repo.logger.Infof("Cloning repository %q to temporary directory %q", repo.cloneURL, repo.path)
+	// Check first if the query param ref is already set
+	q := sourceURL.Query()
 
-	// We need to explicitly specify the reference, otherwise we will get an error:
-	// "fatal: The empty string is not a valid pathspec. Use . instead if you wanted to match all paths"
-	// when updating an existing repository.
-	sourceURL.RawQuery = (url.Values{"ref": []string{"HEAD"}}).Encode()
+	ref := q.Get("ref")
+	if ref != "" {
+		q.Set("ref", "HEAD")
+	}
+
+	sourceURL.RawQuery = q.Encode()
 
-	if err := getter.Get(repo.path, strings.Trim(sourceURL.String(), "/"), getter.WithContext(ctx), getter.WithMode(getter.ClientModeDir)); err != nil {
+	_, err = client.Get(ctx, &getter.Request{
+		Src:     sourceURL.String(),
+		Dst:     repo.path,
+		GetMode: getter.ModeDir,
+	})
+	if err != nil {
+		return err
+	}
+
+	// Create the sentinel file to indicate that the clone is complete
+	f, err := os.Create(filepath.Join(repo.path, cloneCompleteSentinel))
+	if err != nil {
+		return errors.New(err)
+	}
+
+	if err := f.Close(); err != nil {
 		return errors.New(err)
 	}
 
@@ -237,7 +338,7 @@ func (repo *Repo) parseRemoteURL() error {
 	gitConfigPath := filepath.Join(repo.path, ".git", "config")
 
 	if !files.FileExists(gitConfigPath) {
-		return errors.Errorf("the specified path %q is not a git repository", repo.path)
+		return errors.Errorf("the specified path %q is not a git repository (no .git/config file found)", repo.path)
 	}
 
 	repo.logger.Debugf("Parsing git config %q", gitConfigPath)
@@ -280,11 +381,12 @@ func (repo *Repo) gitHeadfile() string {
 func (repo *Repo) parseBranchName() error {
 	data, err := files.ReadFileAsString(repo.gitHeadfile())
 	if err != nil {
-		return errors.Errorf("the specified path %q is not a git repository", repo.path)
+		return errors.Errorf("the specified path %q is not a git repository (no .git/HEAD file found)", repo.path)
 	}
 
 	if match := gitHeadBranchNameReg.FindStringSubmatch(data); len(match) > 0 {
 		repo.BranchName = strings.TrimSpace(match[1])
+
 		return nil
 	}
 

cli/commands/catalog/module/repo_test.go

@@ -64,7 +64,7 @@ func TestFindModules(t *testing.T) {
 
 			ctx := context.Background()
 
-			repo, err := module.NewRepo(ctx, log.New(), testCase.repoPath, "", false)
+			repo, err := module.NewRepo(ctx, log.New(), testCase.repoPath, "", false, false)
 			require.NoError(t, err)
 
 			modules, err := repo.FindModules(ctx)

docs-starlight/src/content/docs/02-features/14-cas.md

@@ -0,0 +1,39 @@
+---
+title: Content Addressable Store (CAS)
+description: Learn how Terragrunt supports deduplication of content using a Content Addressable Store (CAS).
+slug: docs/features/cas
+sidebar:
+  order: 14
+---
+
+Terragrunt supports a Content Addressable Store (CAS) to deduplicate content across multiple Terragrunt configurations. This feature is still experimental and not recommended for general production usage.
+
+At the moment, the only supported use case for the CAS is to speed up catalog cloning. In the future, the CAS can be used to store more content.
+
+To use the CAS, you will need to enable the [cas](/docs/reference/experiments/#cas) experiment.
+
+## Usage
+
+When you enable the `cas` experiment, Terragrunt will automatically use the CAS when cloning any compatible source (right now, only Git repositories).
+
+```hcl
+# root.hcl
+
+catalog {
+  urls = [
+    "[email protected]:acme/modules.git"
+  ]
+}
+```
+
+When Terragrunt clones a repository while using the CAS. If the repository is not found in the CAS, Terragrunt will clone the repository from the original URL and store it in the CAS for future use.
+
+When generating a repository from the CAS, Terragrunt will hard link entries from the CAS to the new repository. This allows Terragrunt to deduplicate content across multiple repositories.
+
+In the event that hard linking fails due to some operating system / host incompatibility with hard links, Terragrunt will fall back to performing copies of the content from the CAS.
+
+## Storage
+
+The CAS is stored in the `~/.cache/terragrunt/cas` directory. This directory can be safely deleted at any time, as Terragrunt will automatically regenerate the CAS as needed.
+
+Avoid partial deletions of the CAS directory without care, as that might result in partially cloned repositories and unexpected behavior.

docs-starlight/src/content/docs/04-reference/04-experiments.md

@@ -166,3 +166,29 @@ To transition `cli-redesign` features to a stable release, the following must be
   - [ ] Add support for `find` with the `exclude` block used to exclude units from the search.
   - [ ] Add integration with `symlinks` experiment to support finding units/stacks via symlinks.
 - [ ] Add support for the `list` command.
+
+### `cas`
+
+Support for Terragrunt Content Addressable Storage (CAS).
+
+#### `cas` - What it does
+
+Allow Terragrunt to store and retrieve state files from a Content Addressable Storage (CAS) system.
+
+At the moment, the CAS is only used to speed up catalog cloning, but in the future, it can be used to store more content.
+
+#### `cas` - How to provide feedback
+
+Share your experience with this feature in the [CAS](https://github.com/gruntwork-io/terragrunt/discussions/3939) Feedback GitHub Discussion.
+Feedback is crucial for ensuring the feature meets real-world use cases. Please include:
+
+- Any bugs or issues encountered (including logs or stack traces if possible).
+- Suggestions for additional improvements or enhancements.
+
+#### `cas` - Criteria for stabilization
+
+To transition the `cas` feature to a stable release, the following must be addressed:
+
+- [x] Add support for storing and retrieving catalog repositories from the CAS.
+- [ ] Add support for storing and retrieving OpenTofu/Terraform modules from the CAS.
+- [ ] Add support for storing and retrieving Unit/Stack configurations from the CAS.