diff --git a/cmd/vela-worker/exec.go b/cmd/vela-worker/exec.go index 3360bdb5..e99f8720 100644 --- a/cmd/vela-worker/exec.go +++ b/cmd/vela-worker/exec.go @@ -47,16 +47,17 @@ func (w *Worker) exec(index int) error { // // https://godoc.org/github.com/go-vela/worker/executor#New _executor, err := executor.New(&executor.Setup{ - Driver: w.Config.Executor.Driver, - LogMethod: w.Config.Executor.LogMethod, - Client: w.VelaClient, - Hostname: w.Config.API.Address.Hostname(), - Runtime: w.Runtime, - Build: item.Build, - Pipeline: item.Pipeline.Sanitize(w.Config.Runtime.Driver), - Repo: item.Repo, - User: item.User, - Version: v.Semantic(), + Driver: w.Config.Executor.Driver, + LogMethod: w.Config.Executor.LogMethod, + MaxLogSize: w.Config.Executor.MaxLogSize, + Client: w.VelaClient, + Hostname: w.Config.API.Address.Hostname(), + Runtime: w.Runtime, + Build: item.Build, + Pipeline: item.Pipeline.Sanitize(w.Config.Runtime.Driver), + Repo: item.Repo, + User: item.User, + Version: v.Semantic(), }) // add the executor to the worker diff --git a/cmd/vela-worker/run.go b/cmd/vela-worker/run.go index 0a9e38cf..4d0fc388 100644 --- a/cmd/vela-worker/run.go +++ b/cmd/vela-worker/run.go @@ -95,8 +95,9 @@ func run(c *cli.Context) error { CheckIn: c.Duration("checkIn"), // executor configuration Executor: &executor.Setup{ - Driver: c.String("executor.driver"), - LogMethod: c.String("executor.log_method"), + Driver: c.String("executor.driver"), + LogMethod: c.String("executor.log_method"), + MaxLogSize: c.Uint("executor.max_log_size"), }, // logger configuration Logger: &Logger{ diff --git a/executor/executor_test.go b/executor/executor_test.go index 55211fda..86e17c5e 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -45,6 +45,7 @@ func TestExecutor_New(t *testing.T) { linux.WithBuild(_build), linux.WithHostname("localhost"), linux.WithLogMethod("byte-chunks"), + linux.WithMaxLogSize(2097152), linux.WithPipeline(_pipeline), linux.WithRepo(_repo), linux.WithRuntime(_runtime), @@ -93,15 +94,16 @@ func TestExecutor_New(t *testing.T) { { failure: false, setup: &Setup{ - Build: _build, - Client: _client, - Driver: constants.DriverLinux, - LogMethod: "byte-chunks", - Pipeline: _pipeline, - Repo: _repo, - Runtime: _runtime, - User: _user, - Version: "v1.0.0", + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + LogMethod: "byte-chunks", + MaxLogSize: 2097152, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + Version: "v1.0.0", }, want: _linux, }, diff --git a/executor/flags.go b/executor/flags.go index 73895019..f889ac6c 100644 --- a/executor/flags.go +++ b/executor/flags.go @@ -49,4 +49,10 @@ var Flags = []cli.Flag{ Usage: "method used to publish logs to the server - options: (byte-chunks|time-chunks)", Value: "byte-chunks", }, + &cli.UintFlag{ + EnvVars: []string{"VELA_EXECUTOR_MAX_LOG_SIZE", "EXECUTOR_MAX_LOG_SIZE"}, + FilePath: "/vela/executor/max_log_size", + Name: "executor.max_log_size", + Usage: "maximum log size (in bytes)", + }, } diff --git a/executor/linux/linux.go b/executor/linux/linux.go index 96248d5c..2d2a5c1e 100644 --- a/executor/linux/linux.go +++ b/executor/linux/linux.go @@ -30,12 +30,13 @@ type ( secret *secretSvc // private fields - init *pipeline.Container - logger *logrus.Entry - logMethod string - build *library.Build - pipeline *pipeline.Build - repo *library.Repo + init *pipeline.Container + logger *logrus.Entry + logMethod string + maxLogSize uint + build *library.Build + pipeline *pipeline.Build + repo *library.Repo // nolint: structcheck,unused // ignore false positives secrets sync.Map services sync.Map diff --git a/executor/linux/opts.go b/executor/linux/opts.go index d23914ff..a4a72f84 100644 --- a/executor/linux/opts.go +++ b/executor/linux/opts.go @@ -59,6 +59,18 @@ func WithLogMethod(method string) Opt { } } +// WithMaxLogSize set the maximum log size (in bytes) in the client. +func WithMaxLogSize(size uint) Opt { + logrus.Trace("configuring maximum log size in linux client") + + return func(c *client) error { + // set the maximum log size in the client + c.maxLogSize = size + + return nil + } +} + // WithHostname sets the hostname in the client. func WithHostname(hostname string) Opt { logrus.Trace("configuring hostname in linux client") diff --git a/executor/linux/opts_test.go b/executor/linux/opts_test.go index 2fc20829..026d9c1c 100644 --- a/executor/linux/opts_test.go +++ b/executor/linux/opts_test.go @@ -109,6 +109,41 @@ func TestLinux_Opt_WithLogMethod(t *testing.T) { } } +func TestLinux_Opt_WithMaxLogSize(t *testing.T) { + // setup tests + tests := []struct { + failure bool + maxLogSize uint + }{ + { + failure: false, + maxLogSize: 2097152, + }, + } + + // run tests + for _, test := range tests { + _engine, err := New( + WithMaxLogSize(test.maxLogSize), + ) + + if test.failure { + if err == nil { + t.Errorf("WithMaxLogSize should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("WithMaxLogSize returned err: %v", err) + } + + if !reflect.DeepEqual(_engine.maxLogSize, test.maxLogSize) { + t.Errorf("WithMaxLogSize is %v, want %v", _engine.maxLogSize, test.maxLogSize) + } + } +} func TestLinux_Opt_WithHostname(t *testing.T) { // setup tests tests := []struct { diff --git a/executor/linux/service.go b/executor/linux/service.go index c60843f5..d9d32e9d 100644 --- a/executor/linux/service.go +++ b/executor/linux/service.go @@ -197,6 +197,13 @@ func (c *client) StreamService(ctx context.Context, ctn *pipeline.Container) err return } + // don't attempt last upload if log size exceeded + if c.maxLogSize > 0 && uint(len(data)) >= c.maxLogSize { + logger.Trace("maximum log size reached") + + return + } + // overwrite the existing log with all bytes // // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.SetData @@ -277,6 +284,13 @@ func (c *client) StreamService(ctx context.Context, ctn *pipeline.Container) err // flush the buffer of logs logs.Reset() } + + // check whether we've reached the maximum log size + if c.maxLogSize > 0 && uint(len(_log.GetData())) >= c.maxLogSize { + logger.Trace("maximum log size reached") + + return + } } } }() @@ -330,6 +344,13 @@ func (c *client) StreamService(ctx context.Context, ctn *pipeline.Container) err // flush the buffer of logs logs.Reset() } + + // check whether we've reached the maximum log size + if c.maxLogSize > 0 && uint(len(_log.GetData())) >= c.maxLogSize { + logger.Trace("maximum log size reached") + + break + } } logger.Info("finished streaming logs") diff --git a/executor/linux/step.go b/executor/linux/step.go index ff6c38b4..adb91d23 100644 --- a/executor/linux/step.go +++ b/executor/linux/step.go @@ -232,6 +232,13 @@ func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error return } + // don't attempt last upload if log size exceeded + if c.maxLogSize > 0 && uint(len(data)) >= c.maxLogSize { + logger.Trace("maximum log size reached") + + return + } + // overwrite the existing log with all bytes // // https://pkg.go.dev/github.com/go-vela/types/library?tab=doc#Log.SetData @@ -312,6 +319,13 @@ func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error // flush the buffer of logs logs.Reset() } + + // check whether we've reached the maximum log size + if c.maxLogSize > 0 && uint(len(_log.GetData())) >= c.maxLogSize { + logger.Trace("maximum log size reached") + + return + } } } }() @@ -365,6 +379,13 @@ func (c *client) StreamStep(ctx context.Context, ctn *pipeline.Container) error // flush the buffer of logs logs.Reset() } + + // check whether we've reached the maximum log size + if c.maxLogSize > 0 && uint(len(_log.GetData())) >= c.maxLogSize { + logger.Trace("maximum log size reached") + + break + } } logger.Info("finished streaming logs") diff --git a/executor/linux/step_test.go b/executor/linux/step_test.go index adee0d87..eb64c85b 100644 --- a/executor/linux/step_test.go +++ b/executor/linux/step_test.go @@ -323,6 +323,10 @@ func TestLinux_StreamStep(t *testing.T) { _build := testBuild() _repo := testRepo() _user := testUser() + _logs := new(library.Log) + + // fill log with bytes + _logs.SetData(make([]byte, 1000)) gin.SetMode(gin.TestMode) @@ -346,6 +350,7 @@ func TestLinux_StreamStep(t *testing.T) { }{ { // init step container failure: false, + logs: _logs, container: &pipeline.Container{ ID: "step_github_octocat_1_init", Directory: "/vela/src/github.com/github/octocat", @@ -358,6 +363,7 @@ func TestLinux_StreamStep(t *testing.T) { }, { // basic step container failure: false, + logs: _logs, container: &pipeline.Container{ ID: "step_github_octocat_1_echo", Directory: "/vela/src/github.com/github/octocat", @@ -370,6 +376,7 @@ func TestLinux_StreamStep(t *testing.T) { }, { // step container with name not found failure: true, + logs: _logs, container: &pipeline.Container{ ID: "step_github_octocat_1_notfound", Directory: "/vela/src/github.com/github/octocat", @@ -382,6 +389,7 @@ func TestLinux_StreamStep(t *testing.T) { }, { // empty step container failure: true, + logs: _logs, container: new(pipeline.Container), }, } @@ -391,6 +399,7 @@ func TestLinux_StreamStep(t *testing.T) { _engine, err := New( WithBuild(_build), WithPipeline(new(pipeline.Build)), + WithMaxLogSize(10), WithRepo(_repo), WithRuntime(_runtime), WithUser(_user), diff --git a/executor/setup.go b/executor/setup.go index 96db863d..52fd8821 100644 --- a/executor/setup.go +++ b/executor/setup.go @@ -32,6 +32,8 @@ type Setup struct { Driver string // specifies the executor method used to publish logs LogMethod string + // specifies the maximum log size + MaxLogSize uint // specifies the executor hostname Hostname string // specifies the executor version @@ -72,6 +74,7 @@ func (s *Setup) Linux() (Engine, error) { return linux.New( linux.WithBuild(s.Build), linux.WithLogMethod(s.LogMethod), + linux.WithMaxLogSize(s.MaxLogSize), linux.WithHostname(s.Hostname), linux.WithPipeline(s.Pipeline), linux.WithRepo(s.Repo), diff --git a/executor/setup_test.go b/executor/setup_test.go index 90588876..a09705bb 100644 --- a/executor/setup_test.go +++ b/executor/setup_test.go @@ -78,6 +78,7 @@ func TestExecutor_Setup_Linux(t *testing.T) { want, err := linux.New( linux.WithBuild(_build), linux.WithLogMethod("byte-chunks"), + linux.WithMaxLogSize(2097152), linux.WithHostname("localhost"), linux.WithPipeline(_pipeline), linux.WithRepo(_repo), @@ -91,16 +92,17 @@ func TestExecutor_Setup_Linux(t *testing.T) { } _setup := &Setup{ - Build: _build, - Client: _client, - Driver: constants.DriverLinux, - LogMethod: "byte-chunks", - Hostname: "localhost", - Pipeline: _pipeline, - Repo: _repo, - Runtime: _runtime, - User: _user, - Version: "v1.0.0", + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + LogMethod: "byte-chunks", + MaxLogSize: 2097152, + Hostname: "localhost", + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, + Version: "v1.0.0", } // run test @@ -226,131 +228,141 @@ func TestExecutor_Setup_Validate(t *testing.T) { }{ { setup: &Setup{ - Build: _build, - Client: _client, - Driver: constants.DriverLinux, - LogMethod: "byte-chunks", - Pipeline: _pipeline, - Repo: _repo, - Runtime: _runtime, - User: _user, + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + LogMethod: "byte-chunks", + MaxLogSize: 2097152, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, }, failure: false, }, { setup: &Setup{ - Build: nil, - Client: _client, - Driver: constants.DriverLinux, - LogMethod: "byte-chunks", - Pipeline: _pipeline, - Repo: _repo, - Runtime: _runtime, - User: _user, + Build: nil, + Client: _client, + Driver: constants.DriverLinux, + LogMethod: "byte-chunks", + MaxLogSize: 2097152, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, }, failure: true, }, { setup: &Setup{ - Build: _build, - Client: nil, - Driver: constants.DriverLinux, - LogMethod: "byte-chunks", - Pipeline: _pipeline, - Repo: _repo, - Runtime: _runtime, - User: _user, + Build: _build, + Client: nil, + Driver: constants.DriverLinux, + LogMethod: "byte-chunks", + MaxLogSize: 2097152, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, }, failure: true, }, { setup: &Setup{ - Build: _build, - Client: _client, - Driver: "", - LogMethod: "byte-chunks", - Pipeline: _pipeline, - Repo: _repo, - Runtime: _runtime, - User: _user, + Build: _build, + Client: _client, + Driver: "", + LogMethod: "byte-chunks", + MaxLogSize: 2097152, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, }, failure: true, }, { setup: &Setup{ - Build: _build, - Client: _client, - Driver: constants.DriverLinux, - LogMethod: "byte-chunks", - Pipeline: nil, - Repo: _repo, - Runtime: _runtime, - User: _user, + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + LogMethod: "byte-chunks", + MaxLogSize: 2097152, + Pipeline: nil, + Repo: _repo, + Runtime: _runtime, + User: _user, }, failure: true, }, { setup: &Setup{ - Build: _build, - Client: _client, - Driver: constants.DriverLinux, - LogMethod: "byte-chunks", - Pipeline: _pipeline, - Repo: nil, - Runtime: _runtime, - User: _user, + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + LogMethod: "byte-chunks", + MaxLogSize: 2097152, + Pipeline: _pipeline, + Repo: nil, + Runtime: _runtime, + User: _user, }, failure: true, }, { setup: &Setup{ - Build: _build, - Client: _client, - Driver: constants.DriverLinux, - LogMethod: "byte-chunks", - Pipeline: _pipeline, - Repo: _repo, - Runtime: nil, - User: _user, + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + LogMethod: "byte-chunks", + MaxLogSize: 2097152, + Pipeline: _pipeline, + Repo: _repo, + Runtime: nil, + User: _user, }, failure: true, }, { setup: &Setup{ - Build: _build, - Client: _client, - Driver: constants.DriverLinux, - LogMethod: "byte-chunks", - Pipeline: _pipeline, - Repo: _repo, - Runtime: _runtime, - User: nil, + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + LogMethod: "byte-chunks", + MaxLogSize: 2097152, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: nil, }, failure: true, }, { setup: &Setup{ - Build: _build, - Client: _client, - Driver: constants.DriverLinux, - LogMethod: "", - Pipeline: _pipeline, - Repo: _repo, - Runtime: _runtime, - User: _user, + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + LogMethod: "", + MaxLogSize: 2097152, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, }, failure: true, }, { setup: &Setup{ - Build: _build, - Client: _client, - Driver: constants.DriverLinux, - LogMethod: "foobar", - Pipeline: _pipeline, - Repo: _repo, - Runtime: _runtime, - User: _user, + Build: _build, + Client: _client, + Driver: constants.DriverLinux, + LogMethod: "foobar", + MaxLogSize: 2097152, + Pipeline: _pipeline, + Repo: _repo, + Runtime: _runtime, + User: _user, }, failure: true, }, diff --git a/router/middleware/executor/executor_test.go b/router/middleware/executor/executor_test.go index 6e64e926..a1839000 100644 --- a/router/middleware/executor/executor_test.go +++ b/router/middleware/executor/executor_test.go @@ -30,14 +30,15 @@ func TestExecutor_Retrieve(t *testing.T) { } want, err := executor.New(&executor.Setup{ - Driver: constants.DriverLinux, - LogMethod: "byte-chunks", - Client: new(vela.Client), - Runtime: _runtime, - Build: new(library.Build), - Pipeline: new(pipeline.Build), - Repo: new(library.Repo), - User: new(library.User), + Driver: constants.DriverLinux, + LogMethod: "byte-chunks", + MaxLogSize: 2097152, + Client: new(vela.Client), + Runtime: _runtime, + Build: new(library.Build), + Pipeline: new(pipeline.Build), + Repo: new(library.Repo), + User: new(library.User), }) if err != nil { t.Errorf("unable to create executor engine: %v", err) @@ -65,14 +66,15 @@ func TestExecutor_Establish(t *testing.T) { } want, err := executor.New(&executor.Setup{ - Driver: constants.DriverLinux, - LogMethod: "byte-chunks", - Client: new(vela.Client), - Runtime: _runtime, - Build: new(library.Build), - Pipeline: new(pipeline.Build), - Repo: new(library.Repo), - User: new(library.User), + Driver: constants.DriverLinux, + LogMethod: "byte-chunks", + MaxLogSize: 2097152, + Client: new(vela.Client), + Runtime: _runtime, + Build: new(library.Build), + Pipeline: new(pipeline.Build), + Repo: new(library.Repo), + User: new(library.User), }) if err != nil { t.Errorf("unable to create executor engine: %v", err)