Simple includes implementation

cmd/build.go

@@ -15,6 +15,7 @@ import (
 	"github.com/elastic/elastic-package/internal/cobraext"
 	"github.com/elastic/elastic-package/internal/docs"
 	"github.com/elastic/elastic-package/internal/files"
+	"github.com/elastic/elastic-package/internal/includes"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
 )
@@ -72,6 +73,14 @@ func buildCommandAction(cmd *cobra.Command, args []string) error {
 	}
 	logger.Debugf("Use build directory: %s", buildDir)
 
+	included, err := includes.IncludeSharedFiles()
+	if err != nil {
+		return fmt.Errorf("updating included files failed: %w", err)
+	}
+	for _, i := range included {
+		cmd.Printf("%s file copied to: %s\n", filepath.Join(i.Package, i.From), i.To)
+	}
+
 	targets, err := docs.UpdateReadmes(packageRoot)
 	if err != nil {
 		return fmt.Errorf("updating files failed: %w", err)

cmd/lint.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/elastic/elastic-package/internal/cobraext"
 	"github.com/elastic/elastic-package/internal/docs"
+	"github.com/elastic/elastic-package/internal/includes"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/packages"
 	"github.com/elastic/elastic-package/internal/validation"
@@ -45,7 +46,6 @@ func setupLintCommand() *cobraext.Command {
 
 func lintCommandAction(cmd *cobra.Command, args []string) error {
 	cmd.Println("Lint the package")
-
 	readmeFiles, err := docs.AreReadmesUpToDate()
 	if err != nil {
 		for _, f := range readmeFiles {
@@ -58,6 +58,18 @@ func lintCommandAction(cmd *cobra.Command, args []string) error {
 		}
 		return fmt.Errorf("checking readme files are up-to-date failed: %w", err)
 	}
+	includedFiles, err := includes.AreFilesUpToDate()
+	if err != nil {
+		for _, f := range includedFiles {
+			if !f.UpToDate {
+				cmd.Printf("%s is outdated. Rebuild the package with 'elastic-package build'\n%s", f.To, f.Diff)
+			}
+			if f.Error != nil {
+				cmd.Printf("check if %s is up-to-date failed: %s\n", f.To, f.Error)
+			}
+		}
+		return fmt.Errorf("checking included files are up-to-date failed: %w", err)
+	}
 	return nil
 }
 

go.mod

@@ -4,6 +4,8 @@ go 1.23.0
 
 toolchain go1.23.1
 
+replace github.com/elastic/package-spec/v3 => github.com/marc-gr/package-spec/v3 v3.0.0-20250318101845-919190f8ff28
+
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/Masterminds/semver/v3 v3.3.1
@@ -73,7 +75,7 @@ require (
 	github.com/elastic/kbncontent v0.1.4 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.9.0+incompatible // indirect
-	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect

go.sum

@@ -84,14 +84,12 @@ github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul
 github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg=
 github.com/elastic/kbncontent v0.1.4 h1:GoUkJkqkn2H6iJTnOHcxEqYVVYyjvcebLQVaSR1aSvU=
 github.com/elastic/kbncontent v0.1.4/go.mod h1:kOPREITK9gSJsiw/WKe7QWSO+PRiZMyEFQCw+CMLAHI=
-github.com/elastic/package-spec/v3 v3.3.2 h1:hUoGvVZU19akdx7gyOtSNGVHjWXeN50NDSqtcGxXtJk=
-github.com/elastic/package-spec/v3 v3.3.2/go.mod h1:worvQ1JmqxT8/SaW3Tijspa5nKR/X7jS6u/hmAxa93w=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls=
 github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
-github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
-github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
+github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
+github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
 github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4=
 github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc=
 github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
@@ -219,6 +217,8 @@ github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg=
 github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/marc-gr/package-spec/v3 v3.0.0-20250318101845-919190f8ff28 h1:abIqzB3NBb971NNZPR8cWa+zNItK4/Z5a76DzCIhwdg=
+github.com/marc-gr/package-spec/v3 v3.0.0-20250318101845-919190f8ff28/go.mod h1:+q7JpjqBFnNVMmh9VAVfZdOxQ3EmdCD+KM8Cg6VhKgg=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=

internal/includes/includes.go

@@ -0,0 +1,166 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package includes
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/elastic/elastic-package/internal/logger"
+	"github.com/elastic/elastic-package/internal/packages"
+	"github.com/elastic/go-ucfg/yaml"
+	"github.com/pmezard/go-difflib/difflib"
+)
+
+// IncludesFileEntry contains a file reference to include.
+type IncludesFileEntry struct {
+	Package  string `config:"package"`
+	From     string `config:"from"`
+	To       string `config:"to"`
+	UpToDate bool   `config:"-"`
+	Diff     string `config:"-"`
+	Error    error  `config:"-"`
+}
+
+type IncludesFile []IncludesFileEntry
+
+// IncludeSharedFiles function collects any necessary files to include in the package.
+func IncludeSharedFiles() (IncludesFile, error) {
+	packageRoot, err := packages.MustFindPackageRoot()
+	if err != nil {
+		return nil, fmt.Errorf("package root not found: %w", err)
+	}
+
+	includesFile, err := readIncludes(packageRoot)
+	if err != nil {
+		return nil, fmt.Errorf("could not read includes.yml: %w", err)
+	}
+
+	for _, f := range includesFile {
+		b, err := collectFile(packageRoot, f)
+		if err != nil {
+			return nil, fmt.Errorf("could not collect file %q: %w", filepath.Join(f.Package, f.From), err)
+		}
+		if err := writeFile(packageRoot, f.To, b); err != nil {
+			return nil, fmt.Errorf("could not write destination file %q: %w", filepath.Join(packageRoot, f.To), err)
+		}
+	}
+	return includesFile, nil
+}
+
+// AreFilesUpToDate function checks if all the included files are up-to-date.
+func AreFilesUpToDate() (IncludesFile, error) {
+	packageRoot, err := packages.MustFindPackageRoot()
+	if err != nil {
+		return nil, fmt.Errorf("package root not found: %w", err)
+	}
+
+	includesFile, err := readIncludes(packageRoot)
+	if err != nil {
+		return nil, fmt.Errorf("could not read includes.yml: %w", err)
+	}
+
+	var outdated bool
+	for i := 0; i < len(includesFile); i++ {
+		f := includesFile[i]
+		uptodate, diff, err := isFileUpToDate(packageRoot, f)
+		if !uptodate || err != nil {
+			includesFile[i].UpToDate = uptodate
+			includesFile[i].Diff = diff
+			includesFile[i].Error = err
+			outdated = true
+		}
+	}
+
+	if outdated {
+		return includesFile, fmt.Errorf("files do not match")
+	}
+	return includesFile, nil
+}
+
+func isFileUpToDate(packageRoot string, includedFile IncludesFileEntry) (bool, string, error) {
+	logger.Debugf("Check if %s is up-to-date", includedFile.To)
+
+	newFile, err := collectFile(packageRoot, includedFile)
+	if err != nil {
+		return false, "", err
+	}
+
+	existing, found, err := readFile(packageRoot, includedFile.To)
+	if err != nil {
+		return false, "", fmt.Errorf("reading file failed: %w", err)
+	}
+	if !found {
+		return false, "", nil
+	}
+	if bytes.Equal(existing, newFile) {
+		return true, "", nil
+	}
+	var buf bytes.Buffer
+	err = difflib.WriteUnifiedDiff(&buf, difflib.UnifiedDiff{
+		A:        difflib.SplitLines(string(existing)),
+		B:        difflib.SplitLines(string(newFile)),
+		FromFile: "want",
+		ToFile:   "got",
+		Context:  1,
+	})
+	return false, buf.String(), err
+}
+
+func collectFile(packageRoot string, includedFile IncludesFileEntry) ([]byte, error) {
+	var filePath string
+	if includedFile.Package != "" {
+		filePath = filepath.Join("..", includedFile.Package, includedFile.From)
+	} else {
+		filePath = filepath.Join(packageRoot, includedFile.From)
+	}
+	b, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+func writeFile(packageRoot, to string, b []byte) error {
+	filePath := filepath.Join(packageRoot, to)
+	return os.WriteFile(filePath, b, 0644)
+}
+
+func readIncludes(packageRoot string) (IncludesFile, error) {
+	includesPath := filepath.Join(packageRoot, "_dev", "shared", "includes.yml")
+
+	b, err := os.ReadFile(includesPath)
+	if err != nil && errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+
+	cfg, err := yaml.NewConfig(b)
+	if err != nil {
+		return nil, fmt.Errorf("could not load includes config: %w", err)
+	}
+
+	var includesFile IncludesFile
+	if err := cfg.Unpack(&includesFile); err != nil {
+		return nil, fmt.Errorf("could not parse includes config: %w", err)
+	}
+	return includesFile, nil
+}
+
+func readFile(packageRoot, filePath string) ([]byte, bool, error) {
+	logger.Debugf("Read existing %s file (package: %s)", filePath, packageRoot)
+
+	includesFilepath := filepath.Join(packageRoot, filePath)
+	b, err := os.ReadFile(includesFilepath)
+	if err != nil && errors.Is(err, os.ErrNotExist) {
+		return nil, false, nil
+	}
+	if err != nil {
+		return nil, false, fmt.Errorf("readfile failed (path: %s): %w", includesFilepath, err)
+	}
+	return b, true, err
+}