feat(templates): add support for stages

Makefile

@@ -310,4 +310,14 @@ spec-version-update:
 #
 # Usage: `make spec`
 .PHONY: spec
-spec: spec-gen spec-version-update spec-validate
+spec: spec-gen spec-version-update spec-validate
+
+# The `lint` target is intended to lint the
+# Go source code with golangci-lint.
+#
+# Usage: `make lint`
+.PHONY: lint
+lint:
+	@echo
+	@echo "### Linting Go Code"
+	@golangci-lint run ./...

api/pipeline.go

@@ -90,13 +90,21 @@ func GetPipeline(ctx *gin.Context) {
 		"user": u.GetName(),
 	}).Infof("reading pipeline for repo %s", r.GetFullName())
 
-	pipeline, _, err := getUnprocessedPipeline(ctx)
+	config, comp, err := getUnprocessedPipeline(ctx)
 	if err != nil {
 		util.HandleError(ctx, http.StatusBadRequest, err)
 
 		return
 	}
 
+	pipeline, err := comp.Parse(config, r.GetPipelineType(), map[string]interface{}{})
+	if err != nil {
+		retErr := fmt.Errorf("unable to validate pipeline configuration for %s: %w", repoName(ctx), err)
+		util.HandleError(ctx, http.StatusBadRequest, retErr)
+
+		return
+	}
+
 	writeOutput(ctx, pipeline)
 }
 
@@ -160,13 +168,21 @@ func GetTemplates(ctx *gin.Context) {
 		"user": u.GetName(),
 	}).Infof("reading templates from pipeline for repo %s", r.GetFullName())
 
-	pipeline, _, err := getUnprocessedPipeline(ctx)
+	config, comp, err := getUnprocessedPipeline(ctx)
 	if err != nil {
 		util.HandleError(ctx, http.StatusBadRequest, err)
 
 		return
 	}
 
+	pipeline, err := comp.Parse(config, r.GetPipelineType(), map[string]interface{}{})
+	if err != nil {
+		retErr := fmt.Errorf("unable to validate pipeline configuration for %s: %w", repoName(ctx), err)
+		util.HandleError(ctx, http.StatusBadRequest, retErr)
+
+		return
+	}
+
 	// create map of templates for response body
 	templates, err := getTemplateLinks(ctx, pipeline.Templates)
 	if err != nil {
@@ -225,6 +241,8 @@ func GetTemplates(ctx *gin.Context) {
 
 // ExpandPipeline represents the API handler to capture and
 // expand a pipeline configuration.
+//
+// nolint: dupl // ignore false positive of duplicate code
 func ExpandPipeline(ctx *gin.Context) {
 	// capture middleware values
 	o := org.Retrieve(ctx)
@@ -240,15 +258,17 @@ func ExpandPipeline(ctx *gin.Context) {
 		"user": u.GetName(),
 	}).Infof("expanding templates from pipeline for repo %s", r.GetFullName())
 
-	pipeline, comp, err := getUnprocessedPipeline(ctx)
+	config, comp, err := getUnprocessedPipeline(ctx)
 	if err != nil {
 		util.HandleError(ctx, http.StatusBadRequest, err)
 
 		return
 	}
 
-	if err := expandPipeline(ctx, pipeline, comp, false); err != nil {
-		util.HandleError(ctx, http.StatusBadRequest, err)
+	pipeline, err := comp.CompileLite(config, true, false)
+	if err != nil {
+		retErr := fmt.Errorf("unable to validate pipeline configuration for %s: %w", repoName(ctx), err)
+		util.HandleError(ctx, http.StatusBadRequest, retErr)
 
 		return
 	}
@@ -315,24 +335,22 @@ func ValidatePipeline(ctx *gin.Context) {
 		"user": u.GetName(),
 	}).Infof("validating pipeline for repo %s", r.GetFullName())
 
-	pipeline, comp, err := getUnprocessedPipeline(ctx)
+	config, comp, err := getUnprocessedPipeline(ctx)
 	if err != nil {
 		util.HandleError(ctx, http.StatusBadRequest, err)
 
 		return
 	}
 
+	template := false
+
 	// check optional template query parameter
 	if ok, _ := strconv.ParseBool(ctx.DefaultQuery("template", "true")); ok {
-		if err := expandPipeline(ctx, pipeline, comp, false); err != nil {
-			util.HandleError(ctx, http.StatusBadRequest, err)
-
-			return
-		}
+		template = true
 	}
 
-	// validate the yaml configuration
-	if err = comp.Validate(pipeline); err != nil {
+	pipeline, err := comp.CompileLite(config, template, false)
+	if err != nil {
 		retErr := fmt.Errorf("unable to validate pipeline configuration for %s: %w", repoName(ctx), err)
 		util.HandleError(ctx, http.StatusBadRequest, retErr)
 
@@ -387,6 +405,7 @@ func ValidatePipeline(ctx *gin.Context) {
 
 // CompilePipeline represents the API handler to capture,
 // expand and compile a pipeline configuration.
+// nolint: dupl // ignore false positive of duplicate code
 func CompilePipeline(ctx *gin.Context) {
 	// capture middleware values
 	o := org.Retrieve(ctx)
@@ -402,21 +421,15 @@ func CompilePipeline(ctx *gin.Context) {
 		"user": u.GetName(),
 	}).Infof("compiling pipeline for repo %s", r.GetFullName())
 
-	pipeline, comp, err := getUnprocessedPipeline(ctx)
+	config, comp, err := getUnprocessedPipeline(ctx)
 	if err != nil {
 		util.HandleError(ctx, http.StatusBadRequest, err)
 
 		return
 	}
 
-	if err := expandPipeline(ctx, pipeline, comp, true); err != nil {
-		util.HandleError(ctx, http.StatusBadRequest, err)
-
-		return
-	}
-
-	// validate the yaml configuration
-	if err = comp.Validate(pipeline); err != nil {
+	pipeline, err := comp.CompileLite(config, true, true)
+	if err != nil {
 		retErr := fmt.Errorf("unable to validate pipeline configuration for %s: %w", repoName(ctx), err)
 		util.HandleError(ctx, http.StatusBadRequest, retErr)
 
@@ -426,8 +439,9 @@ func CompilePipeline(ctx *gin.Context) {
 	writeOutput(ctx, pipeline)
 }
 
-// getUnprocessedPipeline retrieves the unprocessed pipeline from a given context.
-func getUnprocessedPipeline(ctx *gin.Context) (*yaml.Build, compiler.Engine, error) {
+// getUnprocessedPipeline retrieves the unprocessed pipeline from a given context
+// and creates an instance of the compiler with metadata.
+func getUnprocessedPipeline(ctx *gin.Context) ([]byte, compiler.Engine, error) {
 	// capture middleware values
 	meta := ctx.MustGet("metadata").(*types.Metadata)
 	repo := repo.Retrieve(ctx)
@@ -453,12 +467,7 @@ func getUnprocessedPipeline(ctx *gin.Context) (*yaml.Build, compiler.Engine, err
 		WithRepo(repo).
 		WithUser(user)
 
-	pipeline, err := comp.Parse(config)
-	if err != nil {
-		return nil, nil, fmt.Errorf("unable to parse pipeline configuration for %s: %w", repoName(ctx), err)
-	}
-
-	return pipeline, comp, nil
+	return config, comp, nil
 }
 
 // getTemplateLinks helper function that retrieves source provider links
@@ -529,44 +538,3 @@ func writeOutput(ctx *gin.Context, pipeline interface{}) {
 		ctx.YAML(http.StatusOK, pipeline)
 	}
 }
-
-// expandPipeline uses a given pipeline and compiler to expand stages and steps
-// in the pipeline along with optionally substituting the environmental variables.
-func expandPipeline(ctx *gin.Context, pipeline *yaml.Build, comp compiler.Engine, substituteEnv bool) error {
-	// create map of templates for easy lookup
-	templates := pipeline.Templates.Map()
-
-	var err error
-
-	if len(pipeline.Stages) > 0 {
-		// inject the templates into the stages
-		pipeline.Stages, pipeline.Secrets, pipeline.Services, pipeline.Environment, err = comp.ExpandStages(pipeline, templates)
-		if err != nil {
-			return fmt.Errorf("unable to expand stages in pipeline configuration for %s: %w", repoName(ctx), err)
-		}
-
-		if substituteEnv {
-			// inject the substituted environment variables into the stages
-			pipeline.Stages, err = comp.SubstituteStages(pipeline.Stages)
-			if err != nil {
-				return fmt.Errorf("unable to substitute stages in pipeline configuration for %s: %w", repoName(ctx), err)
-			}
-		}
-	} else {
-		// inject the templates into the steps
-		pipeline.Steps, pipeline.Secrets, pipeline.Services, pipeline.Environment, err = comp.ExpandSteps(pipeline, templates)
-		if err != nil {
-			return fmt.Errorf("unable to expand steps in pipeline configuration for %s: %w", repoName(ctx), err)
-		}
-
-		if substituteEnv {
-			// inject the substituted environment variables into the steps
-			pipeline.Steps, err = comp.SubstituteSteps(pipeline.Steps)
-			if err != nil {
-				return fmt.Errorf("unable to substitute steps in pipeline configuration for %s: %w", repoName(ctx), err)
-			}
-		}
-	}
-
-	return nil
-}

compiler/engine.go

@@ -22,13 +22,18 @@ type Engine interface {
 	// Parse internally to convert the object to a yaml configuration.
 	Compile(interface{}) (*pipeline.Build, error)
 
+	// CompileLite defines a function that produces an light executable
+	// representation of a pipeline from an object. This calls
+	// Parse internally to convert the object to a yaml configuration.
+	CompileLite(interface{}, bool, bool) (*yaml.Build, error)
+
 	// Duplicate defines a function that
 	// creates a clone of the Engine.
 	Duplicate() Engine
 
 	// Parse defines a function that converts
 	// an object to a yaml configuration.
-	Parse(interface{}) (*yaml.Build, error)
+	Parse(interface{}, string, map[string]interface{}) (*yaml.Build, error)
 
 	// ParseRaw defines a function that converts
 	// an object to a string.
@@ -66,12 +71,10 @@ type Engine interface {
 
 	// ExpandStages defines a function that injects the template
 	// for each templated step in every stage in a yaml configuration.
-
-	ExpandStages(*yaml.Build, map[string]*yaml.Template) (yaml.StageSlice, yaml.SecretSlice, yaml.ServiceSlice, raw.StringSliceMap, error)
+	ExpandStages(*yaml.Build, map[string]*yaml.Template) (*yaml.Build, error)
 	// ExpandSteps defines a function that injects the template
 	// for each templated step in a yaml configuration.
-
-	ExpandSteps(*yaml.Build, map[string]*yaml.Template) (yaml.StepSlice, yaml.SecretSlice, yaml.ServiceSlice, raw.StringSliceMap, error)
+	ExpandSteps(*yaml.Build, map[string]*yaml.Template) (*yaml.Build, error)
 
 	// Init Compiler Interface Functions