Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
04fb45d
Simple includes implementation
marc-gr Mar 18, 2025
f778462
Second iteration
marc-gr Mar 18, 2025
f24f948
Use link files instead of includes file
marc-gr Mar 19, 2025
52132f5
Collect included pipelines in tests and benchmarks
marc-gr Mar 20, 2025
fc8f054
Create complete path if not exist
marc-gr Mar 25, 2025
268e23e
Add test package
marc-gr Mar 25, 2025
4b6dcb0
Add links commands
marc-gr Mar 25, 2025
0fb950c
Improve links commands
marc-gr Mar 26, 2025
d4f4c36
List also if not in package
marc-gr Mar 26, 2025
f044059
Reorganize code
marc-gr Mar 27, 2025
e9196c0
Merge remote-tracking branch 'upstream/main' into feat/includes
marc-gr Mar 27, 2025
44a1d1a
go mod tidy
marc-gr Mar 27, 2025
afc4b26
Only read entire file when copying
marc-gr Mar 28, 2025
53eead0
Add unit tests
marc-gr Mar 31, 2025
c2cdf7b
Always copy file on build
marc-gr Mar 31, 2025
eb8b05c
remove unused function
marc-gr Mar 31, 2025
0fdd2df
Use package in spec
marc-gr Apr 2, 2025
a4322e5
Use package spec
marc-gr Apr 2, 2025
663ec29
Update links usage
marc-gr Apr 10, 2025
c10bdee
Merge remote-tracking branch 'upstream/main' into feat/includes
marc-gr Apr 10, 2025
6c94355
replace package-spec
marc-gr Apr 10, 2025
ed5a641
Update readme
marc-gr Apr 10, 2025
3438ac4
Merge remote-tracking branch 'upstream/main' into feat/includes
marc-gr Jun 23, 2025
f30f574
Remove go.mod replace
marc-gr Jun 23, 2025
6526118
Secure linked files: fix path traversal vulnerabilities and improve t…
marc-gr Jun 26, 2025
1e50f26
Improve linkedfiles API with convenience functions and structured res…
marc-gr Jun 26, 2025
efd9c9f
fix test cleanup
marc-gr Jun 26, 2025
3c56a23
Use spec 3.4.0 for test package
marc-gr Jun 26, 2025
ebf7c56
Fix shared folder placement
marc-gr Jun 26, 2025
1880bcd
Close root usage on cleanup
marc-gr Jun 26, 2025
e0e8754
close root
marc-gr Jun 26, 2025
b450f33
handle paths better for linksfs
marc-gr Jun 27, 2025
83478ac
Enhance LinksFS security and path handling
marc-gr Jun 27, 2025
c091ed0
lint
marc-gr Jun 27, 2025
0d70806
Merge remote-tracking branch 'upstream/main' into feat/includes
marc-gr Jun 27, 2025
1fb6aa5
Merge remote-tracking branch 'upstream/main' into feat/includes
marc-gr Jun 30, 2025
12a0bc1
Restore LinksFS support lost in upstream merge
marc-gr Jun 30, 2025
bd5b9b8
Use LinksFS as the source for operations
marc-gr Jul 8, 2025
7668010
Merge remote-tracking branch 'upstream/main' into feat/includes
marc-gr Jul 8, 2025
d4d7634
make linksfs work with relative paths from root and wd
marc-gr Jul 8, 2025
2ab1703
check
marc-gr Jul 8, 2025
70e6bc2
remove err declaration
marc-gr Jul 9, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions cmd/links.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,12 @@ func linksCheckCommandAction(cmd *cobra.Command, args []string) error {
return fmt.Errorf("reading current working directory failed: %w", err)
}

linkedFiles, err := files.CheckLinkedFiles(pwd)
linksFS, err := files.CreateLinksFSFromPath(pwd)
if err != nil {
return fmt.Errorf("creating links filesystem failed: %w", err)
}

linkedFiles, err := linksFS.CheckLinkedFiles()
if err != nil {
return fmt.Errorf("checking linked files are up-to-date failed: %w", err)
}
Expand Down Expand Up @@ -90,7 +95,12 @@ func linksUpdateCommandAction(cmd *cobra.Command, args []string) error {
return fmt.Errorf("reading current working directory failed: %w", err)
}

updatedLinks, err := files.UpdateLinkedFiles(pwd)
linksFS, err := files.CreateLinksFSFromPath(pwd)
if err != nil {
return fmt.Errorf("creating links filesystem failed: %w", err)
}

updatedLinks, err := linksFS.UpdateLinkedFiles()
if err != nil {
return fmt.Errorf("updating linked files checksums failed: %w", err)
}
Expand Down Expand Up @@ -125,7 +135,12 @@ func linksListCommandAction(cmd *cobra.Command, args []string) error {
return fmt.Errorf("reading current working directory failed: %w", err)
}

byPackage, err := files.ListLinkedFilesByPackage(pwd)
linksFS, err := files.CreateLinksFSFromPath(pwd)
if err != nil {
return fmt.Errorf("creating links filesystem failed: %w", err)
}

byPackage, err := linksFS.ListLinkedFilesByPackage()
if err != nil {
return fmt.Errorf("listing linked packages failed: %w", err)
}
Expand Down
7 changes: 6 additions & 1 deletion internal/builder/packages.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,12 @@ func BuildPackage(ctx context.Context, options BuildOptions) (string, error) {
}

logger.Debug("Include linked files")
links, err := files.IncludeLinkedFilesFromPath(options.PackageRoot, destinationDir)
linksFS, err := files.CreateLinksFSFromPath(options.PackageRoot)
if err != nil {
return "", fmt.Errorf("creating links filesystem failed: %w", err)
}

links, err := linksFS.IncludeLinkedFiles(destinationDir)
if err != nil {
return "", fmt.Errorf("including linked files failed: %w", err)
}
Expand Down
148 changes: 63 additions & 85 deletions internal/files/linkedfiles.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,70 +28,6 @@ type PackageLinks struct {
Links []string
}

// CheckLinkedFiles checks if all linked files in the given directory are up-to-date.
// Returns a list of outdated links that need updating.
func CheckLinkedFiles(fromDir string) ([]Link, error) {
repoRoot, err := FindRepositoryRootDirectory()
if err != nil {
return nil, fmt.Errorf("finding repository root: %w", err)
}

root, err := os.OpenRoot(repoRoot)
if err != nil {
return nil, fmt.Errorf("opening repository root: %w", err)
}

return AreLinkedFilesUpToDate(root, fromDir)
}

// UpdateLinkedFiles updates the checksums of all outdated linked files in the given directory.
// Returns a list of links that were updated.
func UpdateLinkedFiles(fromDir string) ([]Link, error) {
repoRoot, err := FindRepositoryRootDirectory()
if err != nil {
return nil, fmt.Errorf("finding repository root: %w", err)
}

root, err := os.OpenRoot(repoRoot)
if err != nil {
return nil, fmt.Errorf("opening repository root: %w", err)
}

return UpdateLinkedFilesChecksums(root, fromDir)
}

// IncludeLinkedFilesFromPath copies all linked files from the source directory to the target directory.
// This is used during package building to include linked files in the build output.
func IncludeLinkedFilesFromPath(fromDir, toDir string) ([]Link, error) {
repoRoot, err := FindRepositoryRootDirectory()
if err != nil {
return nil, fmt.Errorf("finding repository root: %w", err)
}

root, err := os.OpenRoot(repoRoot)
if err != nil {
return nil, fmt.Errorf("opening repository root: %w", err)
}

return IncludeLinkedFiles(root, fromDir, toDir)
}

// ListLinkedFilesByPackage returns a mapping of packages to their linked files that reference
// files from the given directory.
func ListLinkedFilesByPackage(fromDir string) ([]PackageLinks, error) {
repoRoot, err := FindRepositoryRootDirectory()
if err != nil {
return nil, fmt.Errorf("finding repository root: %w", err)
}

root, err := os.OpenRoot(repoRoot)
if err != nil {
return nil, fmt.Errorf("opening repository root: %w", err)
}

return LinkedFilesByPackageFrom(root, fromDir)
}

// CreateLinksFSFromPath creates a LinksFS for the given directory within the repository.
func CreateLinksFSFromPath(workDir string) (*LinksFS, error) {
repoRoot, err := FindRepositoryRootDirectory()
Expand Down Expand Up @@ -124,17 +60,35 @@ type LinksFS struct {
func NewLinksFS(repoRoot *os.Root, workDir string) (*LinksFS, error) {
// Ensure workDir is absolute for os.DirFS
var absWorkDir string
var err error
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit. Why is it needed to define this error?


if filepath.IsAbs(workDir) {
absWorkDir = workDir
} else {
// First try: assume workDir is relative to the repository root
absWorkDir = filepath.Join(repoRoot.Name(), workDir)

// Check if path exists
if _, err := os.Stat(absWorkDir); os.IsNotExist(err) {
// Second try: path might be relative to current working directory
currentDir, err := os.Getwd()
if err == nil {
alternativePath := filepath.Join(currentDir, workDir)
if _, err := os.Stat(alternativePath); err == nil {
// If this path exists, use it instead
absWorkDir = alternativePath
logger.Debugf("Using path relative to current working directory: %s", absWorkDir)
}
}
}
}

// Validate that workDir is within the repository root
inRoot, err := pathIsInRepositoryRoot(repoRoot, absWorkDir)
if err != nil {
return nil, fmt.Errorf("could not validate workDir %s: %w", absWorkDir, err)
}

if !inRoot {
return nil, fmt.Errorf("workDir %s is outside the repository root %s", absWorkDir, repoRoot.Name())
}
Expand Down Expand Up @@ -165,7 +119,7 @@ func (lfs *LinksFS) Open(name string) (fs.File, error) {
// Since workDir is expected to be absolute, we can directly join
linkFilePath := filepath.Join(lfs.workDir, relativeName)

l, err := NewLinkedFile(lfs.repoRoot, linkFilePath)
l, err := newLinkedFile(lfs.repoRoot, linkFilePath)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -196,6 +150,30 @@ func (lfs *LinksFS) ReadFile(name string) ([]byte, error) {
return io.ReadAll(f)
}

// CheckLinkedFiles checks if all linked files in the directory are up-to-date.
// Returns a list of outdated links that need updating.
func (lfs *LinksFS) CheckLinkedFiles() ([]Link, error) {
return areLinkedFilesUpToDate(lfs.repoRoot, lfs.workDir)
}

// UpdateLinkedFiles updates the checksums of all outdated linked files in the directory.
// Returns a list of links that were updated.
func (lfs *LinksFS) UpdateLinkedFiles() ([]Link, error) {
return updateLinkedFilesChecksums(lfs.repoRoot, lfs.workDir)
}

// IncludeLinkedFiles copies all linked files from the source directory to the target directory.
// This is used during package building to include linked files in the build output.
func (lfs *LinksFS) IncludeLinkedFiles(toDir string) ([]Link, error) {
return includeLinkedFiles(lfs.repoRoot, lfs.workDir, toDir)
}

// ListLinkedFilesByPackage returns a mapping of packages to their linked files that reference
// files from the given directory.
func (lfs *LinksFS) ListLinkedFilesByPackage() ([]PackageLinks, error) {
return linkedFilesByPackageFrom(lfs.repoRoot, lfs.workDir)
}

// A Link represents a linked file.
// It contains the path to the link file, the checksum of the link file,
// the path to the included file, and the checksum of the included file contents.
Expand All @@ -215,7 +193,7 @@ type Link struct {
}

// NewLinkedFile creates a new Link from the given link file path.
func NewLinkedFile(root *os.Root, linkFilePath string) (Link, error) {
func newLinkedFile(root *os.Root, linkFilePath string) (Link, error) {
var l Link
l.WorkDir = filepath.Dir(linkFilePath)
if linkPackageRoot, _, _ := packages.FindPackageRootFrom(l.WorkDir); linkPackageRoot != "" {
Expand Down Expand Up @@ -284,9 +262,9 @@ func NewLinkedFile(root *os.Root, linkFilePath string) (Link, error) {
return l, nil
}

// UpdateChecksum function updates the checksum of the linked file.
// updateChecksum function updates the checksum of the linked file.
// It returns true if the checksum was updated, false if it was already up-to-date.
func (l *Link) UpdateChecksum() (bool, error) {
func (l *Link) updateChecksum() (bool, error) {
if l.UpToDate {
return false, nil
}
Expand Down Expand Up @@ -316,18 +294,18 @@ func (l *Link) TargetFilePath(workDir ...string) string {
return filepath.Join(wd, targetFilePath)
}

// IncludeLinkedFiles function includes linked files from the source
// includeLinkedFiles function includes linked files from the source
// directory to the target directory.
// It returns a slice of Link structs representing the included files.
// It also updates the checksum of the linked files.
// Both directories must be relative to the root.
func IncludeLinkedFiles(root *os.Root, fromDir, toDir string) ([]Link, error) {
links, err := ListLinkedFiles(root, fromDir)
func includeLinkedFiles(root *os.Root, fromDir, toDir string) ([]Link, error) {
links, err := listLinkedFiles(root, fromDir)
if err != nil {
return nil, fmt.Errorf("including linked files failed: %w", err)
}
for _, l := range links {
if _, err := l.UpdateChecksum(); err != nil {
if _, err := l.updateChecksum(); err != nil {
return nil, fmt.Errorf("could not update checksum for file %s: %w", l.LinkFilePath, err)
}
targetFilePath := l.TargetFilePath(toDir)
Expand All @@ -343,8 +321,8 @@ func IncludeLinkedFiles(root *os.Root, fromDir, toDir string) ([]Link, error) {
return links, nil
}

// ListLinkedFiles function returns a slice of Link structs representing linked files.
func ListLinkedFiles(root *os.Root, fromDir string) ([]Link, error) {
// listLinkedFiles function returns a slice of Link structs representing linked files.
func listLinkedFiles(root *os.Root, fromDir string) ([]Link, error) {
var linkFiles []string
if err := filepath.Walk(
filepath.FromSlash(fromDir),
Expand All @@ -363,7 +341,7 @@ func ListLinkedFiles(root *os.Root, fromDir string) ([]Link, error) {
links := make([]Link, len(linkFiles))

for i, f := range linkFiles {
l, err := NewLinkedFile(root, filepath.FromSlash(f))
l, err := newLinkedFile(root, filepath.FromSlash(f))
if err != nil {
return nil, fmt.Errorf("could not initialize linked file %s: %w", f, err)
}
Expand Down Expand Up @@ -445,9 +423,9 @@ func writeFile(to string, b []byte) error {
return os.WriteFile(to, b, 0644)
}

// AreLinkedFilesUpToDate function checks if all the linked files are up-to-date.
func AreLinkedFilesUpToDate(root *os.Root, fromDir string) ([]Link, error) {
links, err := ListLinkedFiles(root, fromDir)
// areLinkedFilesUpToDate function checks if all the linked files are up-to-date.
func areLinkedFilesUpToDate(root *os.Root, fromDir string) ([]Link, error) {
links, err := listLinkedFiles(root, fromDir)
if err != nil {
return nil, fmt.Errorf("checking linked files failed: %w", err)
}
Expand All @@ -463,18 +441,18 @@ func AreLinkedFilesUpToDate(root *os.Root, fromDir string) ([]Link, error) {
return outdated, nil
}

// UpdateLinkedFilesChecksums function updates the checksums of the linked files.
// updateLinkedFilesChecksums function updates the checksums of the linked files.
// It returns a slice of updated links.
// If no links were updated, it returns an empty slice.
func UpdateLinkedFilesChecksums(root *os.Root, fromDir string) ([]Link, error) {
links, err := ListLinkedFiles(root, fromDir)
func updateLinkedFilesChecksums(root *os.Root, fromDir string) ([]Link, error) {
links, err := listLinkedFiles(root, fromDir)
if err != nil {
return nil, fmt.Errorf("updating linked files checksums failed: %w", err)
}

var updatedLinks []Link
for _, l := range links {
updated, err := l.UpdateChecksum()
updated, err := l.updateChecksum()
if err != nil {
return nil, fmt.Errorf("updating linked files checksums failed: %w", err)
}
Expand All @@ -486,12 +464,12 @@ func UpdateLinkedFilesChecksums(root *os.Root, fromDir string) ([]Link, error) {
return updatedLinks, nil
}

// LinkedFilesByPackageFrom function returns a slice of PackageLinks containing linked files grouped by package.
// linkedFilesByPackageFrom function returns a slice of PackageLinks containing linked files grouped by package.
// Each PackageLinks contains the package name and a slice of linked file paths.
func LinkedFilesByPackageFrom(root *os.Root, fromDir string) ([]PackageLinks, error) {
func linkedFilesByPackageFrom(root *os.Root, fromDir string) ([]PackageLinks, error) {
// we list linked files from all the root directory
// to check which ones are linked to the 'fromDir' package
links, err := ListLinkedFiles(root, root.Name())
links, err := listLinkedFiles(root, root.Name())
if err != nil {
return nil, fmt.Errorf("listing linked files failed: %w", err)
}
Expand Down
Loading