diff --git a/.env.example b/.env.example index c5bf492e9..fdffdf154 100644 --- a/.env.example +++ b/.env.example @@ -58,4 +58,16 @@ VELA_API=http://localhost:8080 # VELA_SCM_CLIENT= # github client secret from oauth application -# VELA_SCM_SECRET= \ No newline at end of file +# VELA_SCM_SECRET= + +# COMPILER FLAGS +# +# compiler github is whether or not the compiler uses github to pull templates +# +# default: false +# VELA_COMPILER_GITHUB= + +# compiler github url is the url used by the compiler to fetch templates +# +# default: https://github.com +# VELA_COMPILER_GITHUB_URL \ No newline at end of file diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 48b67af59..f9b160a8f 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -197,6 +197,12 @@ func main() { Usage: "modification retries, used by compiler, number of http requires that the modification http request will fail after", Value: 5, }, + &cli.IntFlag{ + EnvVars: []string{"VELA_MAX_TEMPLATE_DEPTH", "MAX_TEMPLATE_DEPTH"}, + Name: "max-template-depth", + Usage: "max template depth, used by compiler, maximum number of templates that can be called in a template chain", + Value: 3, + }, &cli.DurationFlag{ EnvVars: []string{"VELA_WORKER_ACTIVE_INTERVAL", "WORKER_ACTIVE_INTERVAL"}, Name: "worker-active-interval", diff --git a/cmd/vela-server/validate.go b/cmd/vela-server/validate.go index bce8b12fb..cac3b0424 100644 --- a/cmd/vela-server/validate.go +++ b/cmd/vela-server/validate.go @@ -117,5 +117,9 @@ func validateCompiler(c *cli.Context) error { } } + if c.Int("max-template-depth") < 1 { + return fmt.Errorf("max-template-depth (VELA_MAX_TEMPLATE_DEPTH) or (MAX_TEMPLATE_DEPTH) flag must be greater than 0") + } + return nil } diff --git a/compiler/engine.go b/compiler/engine.go index dfe0716c4..5d1d32993 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -73,8 +73,8 @@ type Engine interface { // for each templated step in every stage in a yaml configuration. ExpandStages(*yaml.Build, map[string]*yaml.Template, *pipeline.RuleData) (*yaml.Build, error) // ExpandSteps defines a function that injects the template - // for each templated step in a yaml configuration. - ExpandSteps(*yaml.Build, map[string]*yaml.Template, *pipeline.RuleData) (*yaml.Build, error) + // for each templated step in a yaml configuration with the provided template depth. + ExpandSteps(*yaml.Build, map[string]*yaml.Template, *pipeline.RuleData, int) (*yaml.Build, error) // Init Compiler Interface Functions diff --git a/compiler/native/compile.go b/compiler/native/compile.go index a36ddc113..f74e19bab 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -82,7 +82,7 @@ func (c *client) Compile(v interface{}) (*pipeline.Build, *library.Pipeline, err switch { case p.Metadata.RenderInline: - newPipeline, err := c.compileInline(p, nil) + newPipeline, err := c.compileInline(p, nil, c.TemplateDepth) if err != nil { return nil, _pipeline, err } @@ -117,7 +117,7 @@ func (c *client) CompileLite(v interface{}, template, substitute bool, localTemp _pipeline.SetType(c.repo.GetPipelineType()) if p.Metadata.RenderInline { - newPipeline, err := c.compileInline(p, localTemplates) + newPipeline, err := c.compileInline(p, localTemplates, c.TemplateDepth) if err != nil { return nil, _pipeline, err } @@ -169,7 +169,7 @@ func (c *client) CompileLite(v interface{}, template, substitute bool, localTemp } case len(p.Steps) > 0: // inject the templates into the steps - p, err = c.ExpandSteps(p, templates, nil) + p, err = c.ExpandSteps(p, templates, nil, c.TemplateDepth) if err != nil { return nil, _pipeline, err } @@ -194,10 +194,17 @@ func (c *client) CompileLite(v interface{}, template, substitute bool, localTemp } // compileInline parses and expands out inline pipelines. -func (c *client) compileInline(p *yaml.Build, localTemplates []string) (*yaml.Build, error) { +func (c *client) compileInline(p *yaml.Build, localTemplates []string, depth int) (*yaml.Build, error) { newPipeline := *p newPipeline.Templates = yaml.TemplateSlice{} + // return if max template depth has been reached + if depth == 0 { + retErr := fmt.Errorf("max template depth of %d exceeded", c.TemplateDepth) + + return nil, retErr + } + for _, template := range p.Templates { if c.local { for _, file := range localTemplates { @@ -231,6 +238,14 @@ func (c *client) compileInline(p *yaml.Build, localTemplates []string) (*yaml.Bu return nil, err } + // if template parsed contains a template reference, recurse with decremented depth + if len(parsed.Templates) > 0 && parsed.Metadata.RenderInline { + parsed, err = c.compileInline(parsed, localTemplates, depth-1) + if err != nil { + return nil, err + } + } + switch { case len(parsed.Environment) > 0: for key, value := range parsed.Environment { @@ -276,12 +291,6 @@ func (c *client) compileInline(p *yaml.Build, localTemplates []string) (*yaml.Bu } } - // validate the yaml configuration - err := c.Validate(&newPipeline) - if err != nil { - return nil, err - } - return &newPipeline, nil } @@ -307,7 +316,7 @@ func (c *client) compileSteps(p *yaml.Build, _pipeline *library.Pipeline, tmpls } // inject the templates into the steps - p, err = c.ExpandSteps(p, tmpls, r) + p, err = c.ExpandSteps(p, tmpls, r, c.TemplateDepth) if err != nil { return nil, _pipeline, err } diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index 5ea927b33..67c89f3a2 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -38,6 +38,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -417,6 +418,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -619,6 +621,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -883,6 +886,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -1112,6 +1116,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName(t *testi set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -1231,6 +1236,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName_Inline(t set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -1346,6 +1352,7 @@ func TestNative_Compile_InvalidType(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -1402,6 +1409,7 @@ func TestNative_Compile_Clone(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -1592,6 +1600,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -2161,6 +2170,7 @@ func Test_Compile_Inline(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.String("clone-image", defaultCloneImage, "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ @@ -2360,6 +2370,179 @@ func Test_Compile_Inline(t *testing.T) { }, wantErr: false, }, + { + name: "nested templates", + args: args{ + file: "testdata/inline_nested_template.yml", + }, + want: &pipeline.Build{ + Version: "1", + ID: "__0", + Metadata: pipeline.Metadata{ + Clone: true, + Environment: []string{"steps", "services", "secrets"}, + }, + Stages: []*pipeline.Stage{ + { + Name: "init", + Environment: initEnv, + Steps: pipeline.ContainerSlice{ + &pipeline.Container{ + ID: "__0_init_init", + Directory: "/vela/src/foo//", + Environment: initEnv, + Image: "#init", + Name: "init", + Number: 1, + Pull: "not_present", + }, + }, + }, + { + Name: "clone", + Environment: initEnv, + Steps: pipeline.ContainerSlice{ + &pipeline.Container{ + ID: "__0_clone_clone", + Directory: "/vela/src/foo//", + Environment: initEnv, + Image: defaultCloneImage, + Name: "clone", + Number: 2, + Pull: "not_present", + }, + }, + }, + { + Name: "test", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_test_test", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo from inline", m, ""), + Image: "alpine", + Name: "test", + Pull: "not_present", + Number: 3, + }, + }, + }, + { + Name: "nested_test", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_nested_test_nested_test", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo from inline", m, ""), + Image: "alpine", + Name: "nested_test", + Pull: "not_present", + Number: 4, + }, + }, + }, + { + Name: "nested_golang_foo", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_nested_golang_foo_nested_golang_foo", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo hello from foo", m, ""), + Image: "golang:latest", + Name: "nested_golang_foo", + Pull: "not_present", + Number: 5, + }, + }, + }, + { + Name: "nested_golang_bar", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_nested_golang_bar_nested_golang_bar", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo hello from bar", m, ""), + Image: "golang:latest", + Name: "nested_golang_bar", + Pull: "not_present", + Number: 6, + }, + }, + }, + { + Name: "nested_golang_star", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_nested_golang_star_nested_golang_star", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo hello from star", m, ""), + Image: "golang:latest", + Name: "nested_golang_star", + Pull: "not_present", + Number: 7, + }, + }, + }, + { + Name: "nested_starlark_foo", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_nested_starlark_foo_nested_starlark_build_foo", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo hello from foo", m, ""), + Image: "alpine", + Name: "nested_starlark_build_foo", + Pull: "not_present", + Number: 8, + }, + }, + }, + { + Name: "nested_starlark_bar", + Needs: []string{"clone"}, + Environment: initEnv, + Steps: []*pipeline.Container{ + { + ID: "__0_nested_starlark_bar_nested_starlark_build_bar", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: generateTestEnv("echo hello from bar", m, ""), + Image: "alpine", + Name: "nested_starlark_build_bar", + Pull: "not_present", + Number: 9, + }, + }, + }, + }, + }, + wantErr: false, + }, { name: "root steps", args: args{ @@ -2468,6 +2651,14 @@ func Test_Compile_Inline(t *testing.T) { want: nil, wantErr: true, }, + { + name: "circular template call", + args: args{ + file: "testdata/inline_circular_template.yml", + }, + want: nil, + wantErr: true, + }, { name: "secrets", args: args{ @@ -2944,6 +3135,7 @@ func Test_CompileLite(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) m := &types.Metadata{ diff --git a/compiler/native/expand.go b/compiler/native/expand.go index 102ae9774..5314cf25e 100644 --- a/compiler/native/expand.go +++ b/compiler/native/expand.go @@ -31,7 +31,7 @@ func (c *client) ExpandStages(s *yaml.Build, tmpls map[string]*yaml.Template, r // iterate through all stages for _, stage := range s.Stages { // inject the templates into the steps for the stage - p, err := c.ExpandSteps(&yaml.Build{Steps: stage.Steps, Secrets: s.Secrets, Services: s.Services, Environment: s.Environment}, tmpls, r) + p, err := c.ExpandSteps(&yaml.Build{Steps: stage.Steps, Secrets: s.Secrets, Services: s.Services, Environment: s.Environment}, tmpls, r, c.TemplateDepth) if err != nil { return nil, err } @@ -47,11 +47,18 @@ func (c *client) ExpandStages(s *yaml.Build, tmpls map[string]*yaml.Template, r // ExpandSteps injects the template for each // templated step in a yaml configuration. -func (c *client) ExpandSteps(s *yaml.Build, tmpls map[string]*yaml.Template, r *pipeline.RuleData) (*yaml.Build, error) { +func (c *client) ExpandSteps(s *yaml.Build, tmpls map[string]*yaml.Template, r *pipeline.RuleData, depth int) (*yaml.Build, error) { if len(tmpls) == 0 { return s, nil } + // return if max template depth has been reached + if depth == 0 { + retErr := fmt.Errorf("max template depth of %d exceeded", c.TemplateDepth) + + return s, retErr + } + steps := yaml.StepSlice{} secrets := s.Secrets services := s.Services @@ -117,6 +124,19 @@ func (c *client) ExpandSteps(s *yaml.Build, tmpls map[string]*yaml.Template, r * return s, err } + // if template references other templates, expand again + if len(tmplBuild.Templates) != 0 { + // if the tmplBuild has render_inline but the parent build does not, abort + if tmplBuild.Metadata.RenderInline && !s.Metadata.RenderInline { + return s, fmt.Errorf("cannot use render_inline inside a called template (%s)", step.Template.Name) + } + + tmplBuild, err = c.ExpandSteps(tmplBuild, mapFromTemplates(tmplBuild.Templates), r, depth-1) + if err != nil { + return s, err + } + } + // loop over secrets within template for _, secret := range tmplBuild.Secrets { found := false diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index c89d23d50..a32a2286d 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -43,6 +43,7 @@ func TestNative_ExpandStages(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) tmpls := map[string]*yaml.Template{ @@ -190,6 +191,7 @@ func TestNative_ExpandSteps(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) testRepo := new(library.Repo) @@ -317,7 +319,7 @@ func TestNative_ExpandSteps(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData)) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) if err != nil { t.Errorf("ExpandSteps_Type%s returned err: %v", test.name, err) } @@ -368,6 +370,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) tmpls := map[string]*yaml.Template{ @@ -582,7 +585,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { ruledata := new(pipeline.RuleData) ruledata.Branch = "main" - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, ruledata) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, ruledata, compiler.TemplateDepth) if err != nil { t.Errorf("ExpandSteps returned err: %v", err) } @@ -626,6 +629,7 @@ func TestNative_ExpandStepsStarlark(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") c := cli.NewContext(nil, set, nil) tmpls := map[string]*yaml.Template{ @@ -669,7 +673,7 @@ func TestNative_ExpandStepsStarlark(t *testing.T) { t.Errorf("Creating new compiler returned err: %v", err) } - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Secrets: yaml.SecretSlice{}, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, new(pipeline.RuleData)) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Secrets: yaml.SecretSlice{}, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, new(pipeline.RuleData), compiler.TemplateDepth) if err != nil { t.Errorf("ExpandSteps returned err: %v", err) } @@ -691,6 +695,345 @@ func TestNative_ExpandStepsStarlark(t *testing.T) { } } +func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.GET("/api/v3/repos/foo/bar/contents/:path", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/template.json") + }) + + engine.GET("/api/v3/repos/faz/baz/contents/:path", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/template-calls-template.json") + }) + + s := httptest.NewServer(engine) + defer s.Close() + + // setup types + set := flag.NewFlagSet("test", 0) + set.Bool("github-driver", true, "doc") + set.String("github-url", s.URL, "doc") + set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") + c := cli.NewContext(nil, set, nil) + + testBuild := new(library.Build) + + testBuild.SetID(1) + testBuild.SetCommit("123abc456def") + + testRepo := new(library.Repo) + + testRepo.SetID(1) + testRepo.SetOrg("foo") + testRepo.SetName("bar") + + tests := []struct { + name string + tmpls map[string]*yaml.Template + }{ + { + name: "Test 1", + tmpls: map[string]*yaml.Template{ + "chain": { + Name: "chain", + Source: "github.example.com/faz/baz/template.yml", + Type: "github", + }, + }, + }, + } + + steps := yaml.StepSlice{ + &yaml.Step{ + Name: "sample", + Template: yaml.StepTemplate{ + Name: "chain", + }, + }, + } + + globalEnvironment := raw.StringSliceMap{ + "foo": "test1", + "bar": "test2", + } + + wantSteps := yaml.StepSlice{ + &yaml.Step{ + Commands: []string{"./gradlew downloadDependencies"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "sample_call template_install", + Pull: "always", + }, + &yaml.Step{ + Commands: []string{"./gradlew check"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "sample_call template_test", + Pull: "always", + }, + &yaml.Step{ + Commands: []string{"./gradlew build"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "sample_call template_build", + Pull: "always", + }, + } + + wantSecrets := yaml.SecretSlice{ + &yaml.Secret{ + Name: "docker_username", + Key: "org/repo/foo/bar", + Engine: "native", + Type: "repo", + Origin: yaml.Origin{}, + }, + &yaml.Secret{ + Name: "foo_password", + Key: "org/repo/foo/password", + Engine: "vault", + Type: "repo", + Origin: yaml.Origin{}, + }, + } + + wantServices := yaml.ServiceSlice{ + &yaml.Service{ + Image: "postgres:12", + Name: "postgres", + Pull: "not_present", + }, + } + + wantEnvironment := raw.StringSliceMap{ + "foo": "test1", + "bar": "test2", + "star": "test3", + } + + // run test + compiler, err := New(c) + if err != nil { + t.Errorf("Creating new compiler returned err: %v", err) + } + + compiler.WithBuild(testBuild).WithRepo(testRepo) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + if err != nil { + t.Errorf("ExpandSteps_Type%s returned err: %v", test.name, err) + } + + if diff := cmp.Diff(build.Steps, wantSteps); diff != "" { + t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff) + } + + if diff := cmp.Diff(build.Secrets, wantSecrets); diff != "" { + t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff) + } + + if diff := cmp.Diff(build.Services, wantServices); diff != "" { + t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff) + } + + if diff := cmp.Diff(build.Environment, wantEnvironment); diff != "" { + t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff) + } + }) + } +} + +func TestNative_ExpandSteps_TemplateCallTemplate_CircularFail(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + engine.GET("/api/v3/repos/bad/design/contents/:path", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/template-calls-itself.json") + }) + + s := httptest.NewServer(engine) + defer s.Close() + + // setup types + set := flag.NewFlagSet("test", 0) + set.Bool("github-driver", true, "doc") + set.String("github-url", s.URL, "doc") + set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") + c := cli.NewContext(nil, set, nil) + + testBuild := new(library.Build) + + testBuild.SetID(1) + testBuild.SetCommit("123abc456def") + + testRepo := new(library.Repo) + + testRepo.SetID(1) + testRepo.SetOrg("foo") + testRepo.SetName("bar") + + tests := []struct { + name string + tmpls map[string]*yaml.Template + }{ + { + name: "Test 1", + tmpls: map[string]*yaml.Template{ + "circle": { + Name: "circle", + Source: "github.example.com/bad/design/template.yml", + Type: "github", + }, + }, + }, + } + + steps := yaml.StepSlice{ + &yaml.Step{ + Name: "sample", + Template: yaml.StepTemplate{ + Name: "circle", + }, + }, + } + + globalEnvironment := raw.StringSliceMap{ + "foo": "test1", + "bar": "test2", + } + + // run test + compiler, err := New(c) + if err != nil { + t.Errorf("Creating new compiler returned err: %v", err) + } + + compiler.WithBuild(testBuild).WithRepo(testRepo) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + if err == nil { + t.Errorf("ExpandSteps_Type%s should have returned an error", test.name) + } + }) + } +} + +func TestNative_ExpandSteps_CallTemplateWithRenderInline(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.GET("/api/v3/repos/:org/:repo/contents/:path", func(c *gin.Context) { + body, err := convertFileToGithubResponse(c.Param("path")) + if err != nil { + t.Error(err) + } + c.JSON(http.StatusOK, body) + }) + + s := httptest.NewServer(engine) + defer s.Close() + + // setup types + set := flag.NewFlagSet("test", 0) + set.Bool("github-driver", true, "doc") + set.String("github-url", s.URL, "doc") + set.String("github-token", "", "doc") + set.Int("max-template-depth", 5, "doc") + c := cli.NewContext(nil, set, nil) + + testBuild := new(library.Build) + + testBuild.SetID(1) + testBuild.SetCommit("123abc456def") + + testRepo := new(library.Repo) + + testRepo.SetID(1) + testRepo.SetOrg("foo") + testRepo.SetName("bar") + + tests := []struct { + name string + tmpls map[string]*yaml.Template + }{ + { + name: "Test 1", + tmpls: map[string]*yaml.Template{ + "render_inline": { + Name: "render_inline", + Source: "github.example.com/github/octocat/nested.yml", + Type: "github", + }, + }, + }, + } + + steps := yaml.StepSlice{ + &yaml.Step{ + Name: "sample", + Template: yaml.StepTemplate{ + Name: "render_inline", + }, + }, + } + + globalEnvironment := raw.StringSliceMap{ + "foo": "test1", + "bar": "test2", + } + + // run test + compiler, err := New(c) + if err != nil { + t.Errorf("Creating new compiler returned err: %v", err) + } + + compiler.WithBuild(testBuild).WithRepo(testRepo) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + if err == nil { + t.Errorf("ExpandSteps_Type%s should have returned an error", test.name) + } + }) + } +} + func TestNative_mapFromTemplates(t *testing.T) { // setup types str := "foo" diff --git a/compiler/native/native.go b/compiler/native/native.go index 61b411838..733a32407 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -32,6 +32,7 @@ type client struct { UsePrivateGithub bool ModificationService ModificationConfig CloneImage string + TemplateDepth int build *library.Build comment string @@ -71,6 +72,8 @@ func New(ctx *cli.Context) (*client, error) { // set the clone image to use for the injected clone step c.CloneImage = ctx.String("clone-image") + c.TemplateDepth = ctx.Int("max-template-depth") + if ctx.Bool("github-driver") { logrus.Tracef("setting up Private GitHub Client for %s", ctx.String("github-url")) // setup private github service @@ -110,6 +113,7 @@ func (c *client) Duplicate() compiler.Engine { cc.UsePrivateGithub = c.UsePrivateGithub cc.ModificationService = c.ModificationService cc.CloneImage = c.CloneImage + cc.TemplateDepth = c.TemplateDepth return cc } diff --git a/compiler/native/testdata/circular.yml b/compiler/native/testdata/circular.yml new file mode 100644 index 000000000..bd8327df3 --- /dev/null +++ b/compiler/native/testdata/circular.yml @@ -0,0 +1,16 @@ +metadata: + render_inline: true + template: true + +templates: + - name: bad + source: github.example.com/github/octocat/inline_circular_template.yml + type: github + +stages: + test: + steps: + - name: test + image: alpine + commands: + - echo from inline \ No newline at end of file diff --git a/compiler/native/testdata/inline_circular_template.yml b/compiler/native/testdata/inline_circular_template.yml new file mode 100644 index 000000000..4b0469c58 --- /dev/null +++ b/compiler/native/testdata/inline_circular_template.yml @@ -0,0 +1,19 @@ +version: "1" + +metadata: + render_inline: true + +templates: + - name: nested + source: github.example.com/github/octocat/circular.yml + type: github + vars: + image: golang:latest + +stages: + test: + steps: + - name: test + image: alpine + commands: + - echo from inline \ No newline at end of file diff --git a/compiler/native/testdata/inline_nested_template.yml b/compiler/native/testdata/inline_nested_template.yml new file mode 100644 index 000000000..0faaeb0c8 --- /dev/null +++ b/compiler/native/testdata/inline_nested_template.yml @@ -0,0 +1,19 @@ +version: "1" + +metadata: + render_inline: true + +templates: + - name: nested + source: github.example.com/github/octocat/nested.yml + type: github + vars: + image: golang:latest + +stages: + test: + steps: + - name: test + image: alpine + commands: + - echo from inline \ No newline at end of file diff --git a/compiler/native/testdata/nested.yml b/compiler/native/testdata/nested.yml new file mode 100644 index 000000000..2d44729d7 --- /dev/null +++ b/compiler/native/testdata/nested.yml @@ -0,0 +1,23 @@ +metadata: + render_inline: true + template: true + +templates: + - name: golang + source: github.example.com/github/octocat/golang_inline_stages.yml + format: golang + type: github + vars: + image: golang:latest + - name: starlark + source: github.example.com/github/octocat/starlark_inline_stages.star + format: starlark + type: github + +stages: + test: + steps: + - name: test + image: alpine + commands: + - echo from inline \ No newline at end of file diff --git a/compiler/native/testdata/template-calls-itself.json b/compiler/native/testdata/template-calls-itself.json new file mode 100644 index 000000000..a930d6780 --- /dev/null +++ b/compiler/native/testdata/template-calls-itself.json @@ -0,0 +1,18 @@ +{ + "type": "file", + "encoding": "base64", + "size": 5362, + "name": "template.yml", + "path": "template.yml", + "content": "dmVyc2lvbjogIjEiCgp0ZW1wbGF0ZXM6CiAgLSBuYW1lOiB0ZXN0CiAgICBzb3VyY2U6IGdpdGh1Yi5leGFtcGxlLmNvbS9iYWQvZGVzaWduL3RlbXBsYXRlLnltbAogICAgdHlwZTogZ2l0aHViCgpzdGVwczoKICAtIG5hbWU6IGNhbGwgdGVtcGxhdGUKICAgIHRlbXBsYXRlOgogICAgICBuYW1lOiB0ZXN0Cg==", + "sha": "3d21ec53a331a6f037a91c368710b99387d012c1", + "url": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", + "git_url": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", + "html_url": "https://github.com/octokit/octokit.rb/blob/master/template.yml", + "download_url": "https://raw.githubusercontent.com/octokit/octokit.rb/master/template.yml", + "_links": { + "git": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", + "self": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", + "html": "https://github.com/octokit/octokit.rb/blob/master/template.yml" + } +} \ No newline at end of file diff --git a/compiler/native/testdata/template-calls-template.json b/compiler/native/testdata/template-calls-template.json new file mode 100644 index 000000000..6fb0be303 --- /dev/null +++ b/compiler/native/testdata/template-calls-template.json @@ -0,0 +1,18 @@ +{ + "type": "file", + "encoding": "base64", + "size": 5362, + "name": "template.yml", + "path": "template.yml", + "content": "dmVyc2lvbjogIjEiCgp0ZW1wbGF0ZXM6CiAgLSBuYW1lOiB0ZXN0CiAgICBzb3VyY2U6IGdpdGh1Yi5leGFtcGxlLmNvbS9mb28vYmFyL3RlbXBsYXRlLnltbAogICAgdHlwZTogZ2l0aHViCgpzdGVwczoKICAtIG5hbWU6IGNhbGwgdGVtcGxhdGUKICAgIHRlbXBsYXRlOgogICAgICBuYW1lOiB0ZXN0CiAgICAgIHZhcnM6CiAgICAgICAgaW1hZ2U6IG9wZW5qZGs6bGF0ZXN0CiAgICAgICAgZW52aXJvbm1lbnQ6ICJ7IEdSQURMRV9VU0VSX0hPTUU6IC5ncmFkbGUsIEdSQURMRV9PUFRTOiAtRG9yZy5ncmFkbGUuZGFlbW9uPWZhbHNlIC1Eb3JnLmdyYWRsZS53b3JrZXJzLm1heD0xIC1Eb3JnLmdyYWRsZS5wYXJhbGxlbD1mYWxzZSB9IgogICAgICAgIHB1bGxfcG9saWN5OiAicHVsbDogdHJ1ZSIK", + "sha": "3d21ec53a331a6f037a91c368710b99387d012c1", + "url": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", + "git_url": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", + "html_url": "https://github.com/octokit/octokit.rb/blob/master/template.yml", + "download_url": "https://raw.githubusercontent.com/octokit/octokit.rb/master/template.yml", + "_links": { + "git": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", + "self": "https://api.github.com/repos/octokit/octokit.rb/contents/template.yml", + "html": "https://github.com/octokit/octokit.rb/blob/master/template.yml" + } +} \ No newline at end of file diff --git a/compiler/template/native/render.go b/compiler/template/native/render.go index 95389fd1d..869e4588a 100644 --- a/compiler/template/native/render.go +++ b/compiler/template/native/render.go @@ -61,7 +61,7 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM config.Steps[index].Name = fmt.Sprintf("%s_%s", name, newStep.Name) } - return &types.Build{Steps: config.Steps, Secrets: config.Secrets, Services: config.Services, Environment: config.Environment}, nil + return &types.Build{Metadata: config.Metadata, Steps: config.Steps, Secrets: config.Secrets, Services: config.Services, Environment: config.Environment, Templates: config.Templates}, nil } // RenderBuild renders the templated build.