diff --git a/executor/linux/step.go b/executor/linux/step.go index ce7be40e..8a340584 100644 --- a/executor/linux/step.go +++ b/executor/linux/step.go @@ -10,6 +10,8 @@ import ( "context" "fmt" "io/ioutil" + "regexp" + "strings" "time" "github.com/go-vela/types/constants" @@ -214,6 +216,8 @@ func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error return err } + secretValues := getSecretValues(ctn) + defer func() { // tail the runtime container rc, err := c.Runtime.TailContainer(ctx, ctn) @@ -239,6 +243,9 @@ func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error return } + // mask secrets in logs before setting them in the database. + data = maskSecrets(data, secretValues) + // overwrite the existing log with all bytes // // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.SetData @@ -305,7 +312,10 @@ func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error // update the existing log with the new bytes // // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData - _log.AppendData(logs.Bytes()) + + data := maskSecrets(logs.Bytes(), secretValues) + + _log.AppendData(data) logger.Debug("appending logs") // send API call to append the logs for the step @@ -362,10 +372,13 @@ func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error if logs.Len() > 1000 { logger.Trace(logs.String()) + // mask secrets before updating logs + data := maskSecrets(logs.Bytes(), secretValues) + // update the existing log with the new bytes // // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.AppendData - _log.AppendData(logs.Bytes()) + _log.AppendData(data) logger.Debug("appending logs") // send API call to append the logs for the step @@ -438,3 +451,39 @@ func (c *client) DestroyStep(ctx context.Context, ctn *pipeline.Container) error return nil } + +// getSecretValues is a helper function that creates a slice of +// secret values that will be used to mask secrets in logs before +// updating the database. +func getSecretValues(ctn *pipeline.Container) []string { + secretValues := []string{} + // gather secrets' values from the environment map for masking + for _, secret := range ctn.Secrets { + s := ctn.Environment[strings.ToUpper(secret.Target)] + // handle multi line secrets from files + s = strings.ReplaceAll(s, "\n", " ") + + // drop any trailing spaces + if strings.HasSuffix(s, " ") { + s = s[:(len(s) - 1)] + } + secretValues = append(secretValues, s) + } + return secretValues +} + +// maskSecrets is a helper function that takes in a byte array +// and a slice of secret values to mask. +func maskSecrets(log []byte, secrets []string) []byte { + strData := string(log) + for _, secret := range secrets { + re := regexp.MustCompile(`\s` + secret + `\s`) + matches := re.FindAllString(strData, -1) + for _, match := range matches { + mask := string(match[0]) + constants.SecretLogMask + string(match[len(match)-1]) + strData = strings.Replace(strData, match, mask, 1) + } + strData = re.ReplaceAllString(strData, constants.SecretLogMask) + } + return []byte(strData) +} diff --git a/executor/linux/step_test.go b/executor/linux/step_test.go index eb64c85b..30f86346 100644 --- a/executor/linux/step_test.go +++ b/executor/linux/step_test.go @@ -6,7 +6,9 @@ package linux import ( "context" + "io/ioutil" "net/http/httptest" + "reflect" "testing" "github.com/gin-gonic/gin" @@ -338,6 +340,7 @@ func TestLinux_StreamStep(t *testing.T) { } _runtime, err := docker.NewMock() + if err != nil { t.Errorf("unable to create runtime engine: %v", err) } @@ -522,3 +525,134 @@ func TestLinux_DestroyStep(t *testing.T) { } } } + +func TestLinux_getSecretValues(t *testing.T) { + fileSecret, err := ioutil.ReadFile("./testdata/step/secret_text.txt") + if err != nil { + t.Errorf("unable to read from test data file secret. Err: %v", err) + } + tests := []struct { + want []string + container *pipeline.Container + }{ + { // no secrets container + want: []string{}, + container: &pipeline.Container{ + ID: "step_github_octocat_1_init", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{"FOO": "bar"}, + Image: "#init", + Name: "init", + Number: 1, + Pull: "not_present", + }, + }, + { // secrets container + want: []string{"secretUser", "secretPass"}, + container: &pipeline.Container{ + ID: "step_github_octocat_1_echo", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{ + "FOO": "bar", + "SECRET_USERNAME": "secretUser", + "SECRET_PASSWORD": "secretPass", + }, + Image: "alpine:latest", + Name: "echo", + Number: 1, + Pull: "not_present", + Secrets: pipeline.StepSecretSlice{ + { + Source: "someSource", + Target: "secret_username", + }, + { + Source: "someOtherSource", + Target: "secret_password", + }, + }, + }, + }, + { // secrets container with file as value + want: []string{"secretUser", "this is a secret"}, + container: &pipeline.Container{ + ID: "step_github_octocat_1_ignorenotfound", + Directory: "/vela/src/github.com/github/octocat", + Environment: map[string]string{ + "FOO": "bar", + "SECRET_USERNAME": "secretUser", + "SECRET_PASSWORD": string(fileSecret), + }, + Image: "alpine:latest", + Name: "ignorenotfound", + Number: 1, + Pull: "not_present", + Secrets: pipeline.StepSecretSlice{ + { + Source: "someSource", + Target: "secret_username", + }, + { + Source: "someOtherSource", + Target: "secret_password", + }, + }, + }, + }, + } + // run tests + for _, test := range tests { + got := getSecretValues(test.container) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("getSecretValues is %v, want %v", got, test.want) + } + } +} + +func TestLinux_maskSecrets(t *testing.T) { + // set up test secrets + sVals := []string{"secret", "bigsecret", "littlesecret", "extrasecret"} + + // set up test logs + s1 := "$ echo $NO_SECRET\nnosecret\n" + s2 := "$ echo $SECRET\nbigsecret\n" + s2Masked := "$ echo $SECRET\n***\n" + s3 := "$ echo $SECRET1\nbigsecret\n$ echo $SECRET2\nlittlesecret\n" + s3Masked := "$ echo $SECRET1\n***\n$ echo $SECRET2\n***\n" + + tests := []struct { + want []byte + log []byte + secrets []string + }{ + { // no secrets in log + want: []byte(s1), + log: []byte(s1), + secrets: sVals, + }, + { // one secret in log + want: []byte(s2Masked), + log: []byte(s2), + secrets: sVals, + }, + { // multiple secrets in log + want: []byte(s3Masked), + log: []byte(s3), + secrets: sVals, + }, + { // empty secrets slice + want: []byte(s3), + log: []byte(s3), + secrets: []string{}, + }, + } + // run tests + for _, test := range tests { + got := maskSecrets(test.log, test.secrets) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("maskSecrets is %v, want %v", string(got), string(test.want)) + } + } +} diff --git a/executor/linux/testdata/step/secret_text.txt b/executor/linux/testdata/step/secret_text.txt new file mode 100644 index 00000000..525086b6 --- /dev/null +++ b/executor/linux/testdata/step/secret_text.txt @@ -0,0 +1,4 @@ +this +is +a +secret \ No newline at end of file diff --git a/go.mod b/go.mod index bbe319ee..4d6d4ced 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/gin-gonic/gin v1.7.7 github.com/go-vela/sdk-go v0.11.0 github.com/go-vela/server v0.11.1-0.20211213155322-eeba06d5ce06 - github.com/go-vela/types v0.11.1-0.20211117152001-4dc404f4aabc + github.com/go-vela/types v0.11.1-0.20211221194436-28210cfa70c9 github.com/google/go-cmp v0.5.6 github.com/joho/godotenv v1.4.0 github.com/opencontainers/image-spec v1.0.2 diff --git a/go.sum b/go.sum index 8530f3e0..6e2dafe9 100644 --- a/go.sum +++ b/go.sum @@ -207,8 +207,9 @@ github.com/go-vela/server v0.11.0/go.mod h1:0phuhEP09iKIiNKpO+cfOa6qU+epgr9Oon1M github.com/go-vela/server v0.11.1-0.20211213155322-eeba06d5ce06 h1:5a2t2rh2/zD/+NMVrGw9UyILryaF9naG4cGF+WDWuj4= github.com/go-vela/server v0.11.1-0.20211213155322-eeba06d5ce06/go.mod h1:CG7MFRFVZ4s2ov4B2XRJles4R+vLD+3AMQw7O9qzk1c= github.com/go-vela/types v0.11.0/go.mod h1:8Oml/G1ATFTJsKdsIsstUuHVLsUv7pl6+EiIyOaUqH0= -github.com/go-vela/types v0.11.1-0.20211117152001-4dc404f4aabc h1:5BJtsCPpi0jlw9XEFafcUo/u+kdEA7N9bp7bBW9/hjA= github.com/go-vela/types v0.11.1-0.20211117152001-4dc404f4aabc/go.mod h1:W00S1BayYQhCVqI4GuuhGjg173MOfU9UvK3JEDCr1aw= +github.com/go-vela/types v0.11.1-0.20211221194436-28210cfa70c9 h1:wvbQB5W9P5F9etlG3T0bLfJd8Ct/SYWnCKCysqF6n1w= +github.com/go-vela/types v0.11.1-0.20211221194436-28210cfa70c9/go.mod h1:W00S1BayYQhCVqI4GuuhGjg173MOfU9UvK3JEDCr1aw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=