Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
04fb45d
Simple includes implementation
marc-gr Mar 18, 2025
f778462
Second iteration
marc-gr Mar 18, 2025
f24f948
Use link files instead of includes file
marc-gr Mar 19, 2025
52132f5
Collect included pipelines in tests and benchmarks
marc-gr Mar 20, 2025
fc8f054
Create complete path if not exist
marc-gr Mar 25, 2025
268e23e
Add test package
marc-gr Mar 25, 2025
4b6dcb0
Add links commands
marc-gr Mar 25, 2025
0fb950c
Improve links commands
marc-gr Mar 26, 2025
d4f4c36
List also if not in package
marc-gr Mar 26, 2025
f044059
Reorganize code
marc-gr Mar 27, 2025
e9196c0
Merge remote-tracking branch 'upstream/main' into feat/includes
marc-gr Mar 27, 2025
44a1d1a
go mod tidy
marc-gr Mar 27, 2025
afc4b26
Only read entire file when copying
marc-gr Mar 28, 2025
53eead0
Add unit tests
marc-gr Mar 31, 2025
c2cdf7b
Always copy file on build
marc-gr Mar 31, 2025
eb8b05c
remove unused function
marc-gr Mar 31, 2025
0fdd2df
Use package in spec
marc-gr Apr 2, 2025
a4322e5
Use package spec
marc-gr Apr 2, 2025
663ec29
Update links usage
marc-gr Apr 10, 2025
c10bdee
Merge remote-tracking branch 'upstream/main' into feat/includes
marc-gr Apr 10, 2025
6c94355
replace package-spec
marc-gr Apr 10, 2025
ed5a641
Update readme
marc-gr Apr 10, 2025
3438ac4
Merge remote-tracking branch 'upstream/main' into feat/includes
marc-gr Jun 23, 2025
f30f574
Remove go.mod replace
marc-gr Jun 23, 2025
6526118
Secure linked files: fix path traversal vulnerabilities and improve t…
marc-gr Jun 26, 2025
1e50f26
Improve linkedfiles API with convenience functions and structured res…
marc-gr Jun 26, 2025
efd9c9f
fix test cleanup
marc-gr Jun 26, 2025
3c56a23
Use spec 3.4.0 for test package
marc-gr Jun 26, 2025
ebf7c56
Fix shared folder placement
marc-gr Jun 26, 2025
1880bcd
Close root usage on cleanup
marc-gr Jun 26, 2025
e0e8754
close root
marc-gr Jun 26, 2025
b450f33
handle paths better for linksfs
marc-gr Jun 27, 2025
83478ac
Enhance LinksFS security and path handling
marc-gr Jun 27, 2025
c091ed0
lint
marc-gr Jun 27, 2025
0d70806
Merge remote-tracking branch 'upstream/main' into feat/includes
marc-gr Jun 27, 2025
1fb6aa5
Merge remote-tracking branch 'upstream/main' into feat/includes
marc-gr Jun 30, 2025
12a0bc1
Restore LinksFS support lost in upstream merge
marc-gr Jun 30, 2025
bd5b9b8
Use LinksFS as the source for operations
marc-gr Jul 8, 2025
7668010
Merge remote-tracking branch 'upstream/main' into feat/includes
marc-gr Jul 8, 2025
d4d7634
make linksfs work with relative paths from root and wd
marc-gr Jul 8, 2025
2ab1703
check
marc-gr Jul 8, 2025
70e6bc2
remove err declaration
marc-gr Jul 9, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Only read entire file when copying
  • Loading branch information
marc-gr committed Mar 28, 2025
commit afc4b26621199c13da659323d256db20ab301a0b
7 changes: 3 additions & 4 deletions internal/builder/linkedfiles.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,15 @@ func IncludeLinkedFiles(fromDir, toDir string) ([]files.Link, error) {
for _, l := range links {
l.ReplaceTargetFilePathDirectory(fromDir, toDir)

if err := files.WriteFile(l.TargetFilePath, l.IncludedFileContents); err != nil {
return nil, fmt.Errorf("could not write file %v: %w", l.TargetFilePath, err)
}

updated, err := l.UpdateChecksum()
if err != nil {
return nil, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err)
}

if updated {
if err := files.CopyFile(l.IncludedFilePath, l.TargetFilePath); err != nil {
return nil, fmt.Errorf("could not write file %v: %w", l.TargetFilePath, err)
}
logger.Debugf("%v included in package", l.TargetFilePath)
}
}
Expand Down
53 changes: 42 additions & 11 deletions internal/files/linkedfiles.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,12 @@ type Link struct {
TargetFilePath string

IncludedFilePath string
IncludedFileContents []byte
IncludedFileContentsChecksum string

UpToDate bool
}

func NewLinkedFile(linkFilePath string) (Link, error) {
func newLinkedFile(linkFilePath string) (Link, error) {
var l Link
firstLine, err := readFirstLine(linkFilePath)
if err != nil {
Expand All @@ -52,6 +51,8 @@ func NewLinkedFile(linkFilePath string) (Link, error) {
return l, nil
}

// UpdateChecksum function updates the checksum of the linked file.
// It returns true if the checksum was updated, false if it was already up-to-date.
func (l *Link) UpdateChecksum() (bool, error) {
if l.UpToDate {
return false, nil
Expand All @@ -63,12 +64,13 @@ func (l *Link) UpdateChecksum() (bool, error) {
return false, fmt.Errorf("checksum is empty for file %v", l.IncludedFilePath)
}
newContent := fmt.Sprintf("%v %v", filepath.ToSlash(l.IncludedFilePath), l.IncludedFileContentsChecksum)
if err := WriteFile(l.LinkFilePath, []byte(newContent)); err != nil {
if err := writeFile(l.LinkFilePath, []byte(newContent)); err != nil {
return false, fmt.Errorf("could not update checksum for file %v: %w", l.LinkFilePath, err)
}
return true, nil
}

// ReplaceTargetFilePathDirectory function replaces the target file path directory.
func (l *Link) ReplaceTargetFilePathDirectory(fromDir, toDir string) {
// if a destination dir is set we replace the source dir with the destination dir
if toDir == "" {
Expand Down Expand Up @@ -100,6 +102,9 @@ func AreLinkedFilesUpToDate(fromDir string) ([]Link, error) {
return outdated, nil
}

// UpdateLinkedFilesChecksums function updates the checksums of the linked files.
// It returns a slice of updated links.
// If no links were updated, it returns an empty slice.
func UpdateLinkedFilesChecksums(fromDir string) ([]Link, error) {
links, err := ListLinkedFiles(fromDir)
if err != nil {
Expand All @@ -120,6 +125,8 @@ func UpdateLinkedFilesChecksums(fromDir string) ([]Link, error) {
return updatedLinks, nil
}

// LinkedFilesByPackageFrom function returns a slice of maps containing linked files grouped by package.
// Each map contains the package name as the key and a slice of linked file paths as the value.
func LinkedFilesByPackageFrom(fromDir string) ([]map[string][]string, error) {
rootPath, err := FindRepositoryRootDirectory()
if err != nil {
Expand Down Expand Up @@ -159,6 +166,7 @@ func LinkedFilesByPackageFrom(fromDir string) ([]map[string][]string, error) {
return byPackage, nil
}

// ListLinkedFiles function returns a slice of Link structs representing linked files.
func ListLinkedFiles(fromDir string) ([]Link, error) {
links, err := getLinksFrom(fromDir)
if err != nil {
Expand All @@ -172,21 +180,44 @@ func ListLinkedFiles(fromDir string) ([]Link, error) {

for i := range links {
l := links[i]
b, cs, err := collectFile(root.FS().(fs.ReadFileFS), l.IncludedFilePath)
cs, err := getLinkedFileChecksum(root.FS().(fs.ReadFileFS), l.IncludedFilePath)
if err != nil {
return nil, fmt.Errorf("could not collect file %v: %w", l.IncludedFilePath, err)
}
if l.LinkChecksum == cs {
links[i].UpToDate = true
}
links[i].IncludedFileContents = b
links[i].IncludedFileContentsChecksum = cs
}

return links, nil
}

func WriteFile(to string, b []byte) error {
func CopyFile(from, to string) error {
from = filepath.FromSlash(from)
source, err := os.Open(from)
if err != nil {
return err
}
defer source.Close()

to = filepath.FromSlash(to)
if _, err := os.Stat(filepath.Dir(to)); os.IsNotExist(err) {
if err := os.MkdirAll(filepath.Dir(to), 0700); err != nil {
return err
}
}
destination, err := os.Create(to)
if err != nil {
return err
}
defer destination.Close()

_, err = io.Copy(destination, source)
return err
}

func writeFile(to string, b []byte) error {
to = filepath.FromSlash(to)
if _, err := os.Stat(filepath.Dir(to)); os.IsNotExist(err) {
if err := os.MkdirAll(filepath.Dir(to), 0700); err != nil {
Expand Down Expand Up @@ -214,7 +245,7 @@ func getLinksFrom(fromDir string) ([]Link, error) {
links := make([]Link, len(linkFiles))

for i, f := range linkFiles {
l, err := NewLinkedFile(f)
l, err := newLinkedFile(f)
if err != nil {
return nil, fmt.Errorf("could not create linked file %v: %w", f, err)
}
Expand All @@ -224,16 +255,16 @@ func getLinksFrom(fromDir string) ([]Link, error) {
return links, nil
}

func collectFile(root fs.ReadFileFS, path string) ([]byte, string, error) {
func getLinkedFileChecksum(root fs.ReadFileFS, path string) (string, error) {
b, err := root.ReadFile(filepath.FromSlash(path))
if err != nil {
return nil, "", err
return "", err
}
cs, err := checksum(b)
if err != nil {
return nil, "", err
return "", err
}
return b, cs, nil
return cs, nil
}

func readFirstLine(filePath string) (string, error) {
Expand Down