From bd25ad966e82435947e9297b4cf082c44339d8bd Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Mon, 27 Feb 2023 12:19:58 -0600 Subject: [PATCH 01/14] feat(worker-availability): extend worker table with 4 fields --- api/metrics.go | 106 ++++++++++++++++++++++-- api/worker.go | 20 +++++ database/worker/create_test.go | 6 +- database/worker/table.go | 37 +++++---- database/worker/update_test.go | 6 +- database/worker/worker_test.go | 18 ++-- router/middleware/worker/worker_test.go | 4 + 7 files changed, 164 insertions(+), 33 deletions(-) diff --git a/api/metrics.go b/api/metrics.go index b5f6fe942..41002a039 100644 --- a/api/metrics.go +++ b/api/metrics.go @@ -10,6 +10,7 @@ import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/database" + "github.com/go-vela/types/constants" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -53,6 +54,19 @@ type MetricsQueryParameters struct { ActiveWorkerCount bool `form:"active_worker_count"` // InactiveWorkerCount represents total number of inactive workers InactiveWorkerCount bool `form:"inactive_worker_count"` + + // UnregisteredWorkerCount represents total number of workers with a status of unregistered + UnregisteredWorkerCount bool `form:"unregistered_worker_count"` + // AvailableWorkerCount represents total number of workers with a status of available, + // where worker RunningBuildIDs.length < worker BuildLimit + AvailableWorkerCount bool `form:"available_worker_count"` + // BusyWorkerCount represents total number of workers with a status of busy, + // where worker BuildLimit == worker RunningBuildIDs.length + BusyWorkerCount bool `form:"busy_worker_count"` + // BusyWorkerCount represents total number of workers with a status of maintenance + MaintenanceWorkerCount bool `form:"maintenance_worker_count"` + // ErrorWorkerCount represents total number of workers with a status of error + ErrorWorkerCount bool `form:"error_worker_count"` } // predefine Prometheus metrics else they will be regenerated @@ -172,6 +186,31 @@ var ( // description: Indicates a request for inactive worker count // type: boolean // default: false +// - in: query +// name: unregistered_worker_count +// description: Indicates a request for unregistered worker count +// type: boolean +// default: false +// - in: query +// name: available_worker_count +// description: Indicates a request for available worker count +// type: boolean +// default: false +// - in: query +// name: busy_worker_count +// description: Indicates a request for busy worker count +// type: boolean +// default: false +// - in: query +// name: maintenance_worker_count +// description: Indicates a request for maintenance worker count +// type: boolean +// default: false +// - in: query +// name: error_worker_count +// description: Indicates a request for error worker count +// type: boolean +// default: false // responses: // '200': // description: Successfully retrieved the Vela metrics @@ -356,14 +395,19 @@ func recordGauges(c *gin.Context) { // add worker metrics var ( - buildLimit int64 - activeWorkers int64 - inactiveWorkers int64 + buildLimit int64 + activeWorkers int64 + inactiveWorkers int64 + unregisteredWorkers int64 + availableWorkers int64 + busyWorkers int64 + maintenanceWorkers int64 + errorWorkers int64 ) // get worker metrics based on request query parameters - // worker_build_limit, active_worker_count, inactive_worker_count - if q.WorkerBuildLimit || q.ActiveWorkerCount || q.InactiveWorkerCount { + // worker_build_limit, active_worker_count, inactive_worker_count, unregistered_worker_count, available_worker_count, busy_worker_count, maintenance_worker_count, error_worker_count + if q.WorkerBuildLimit || q.ActiveWorkerCount || q.InactiveWorkerCount || q.UnregisteredWorkerCount || q.AvailableWorkerCount || q.BusyWorkerCount || q.MaintenanceWorkerCount || q.ErrorWorkerCount { // send API call to capture the workers workers, err := database.FromContext(c).ListWorkers() if err != nil { @@ -372,6 +416,8 @@ func recordGauges(c *gin.Context) { // get the unix time from worker_active_interval ago before := time.Now().UTC().Add(-c.Value("worker_active_interval").(time.Duration)).Unix() + + // active, inactive counts for _, worker := range workers { // check if the worker checked in within the last worker_active_interval if worker.GetLastCheckedIn() >= before { @@ -382,6 +428,31 @@ func recordGauges(c *gin.Context) { } } + // available, busy, maintenance, error counts + for _, worker := range workers { + // check if the worker checked in within the last worker_active_interval + if worker.GetLastCheckedIn() >= before { + + switch worker.GetStatus() { + case constants.WorkerStatusUnregistered: + availableWorkers++ + case constants.WorkerStatusBusy: + busyWorkers++ + case constants.WorkerStatusMaintenance: + maintenanceWorkers++ + case constants.WorkerStatusError: + errorWorkers++ + } + } + } + + // unregistered count + for _, worker := range workers { + if worker.GetStatus() == constants.WorkerStatusUnregistered { + unregisteredWorkers++ + } + } + // apply metrics based on request query parameters // worker_build_limit if q.WorkerBuildLimit { @@ -397,5 +468,30 @@ func recordGauges(c *gin.Context) { if q.InactiveWorkerCount { totals.WithLabelValues("worker", "count", "inactive").Set(float64(inactiveWorkers)) } + + // unregistered_worker_count + if q.UnregisteredWorkerCount { + totals.WithLabelValues("worker", "count", "unregistered").Set(float64(unregisteredWorkers)) + } + + // available_worker_count + if q.AvailableWorkerCount { + totals.WithLabelValues("worker", "count", "available").Set(float64(availableWorkers)) + } + + // busy_worker_count + if q.BusyWorkerCount { + totals.WithLabelValues("worker", "count", "busy").Set(float64(busyWorkers)) + } + + // maintenance_worker_count + if q.MaintenanceWorkerCount { + totals.WithLabelValues("worker", "count", "maintenance").Set(float64(maintenanceWorkers)) + } + + // error_worker_count + if q.ErrorWorkerCount { + totals.WithLabelValues("worker", "count", "error").Set(float64(errorWorkers)) + } } } diff --git a/api/worker.go b/api/worker.go index 4951b182a..442ab86e9 100644 --- a/api/worker.go +++ b/api/worker.go @@ -267,6 +267,26 @@ func UpdateWorker(c *gin.Context) { w.SetActive(input.GetActive()) } + if len(input.GetStatus()) > 0 { + // update status if set + w.SetStatus(input.GetStatus()) + } + + if input.GetLastStatusUpdateAt() > 0 { + // update LastStatusUpdateAt if set + w.SetLastStatusUpdateAt(input.GetLastStatusUpdateAt()) + } + + if len(input.GetRunningBuildIDs()) > 0 { + // update RunningBuildIDs if set + w.SetRunningBuildIDs(input.GetRunningBuildIDs()) + } + + if input.GetLastBuildFinishedAt() > 0 { + // update LastBuildFinishedAt if set + w.SetLastBuildFinishedAt(input.GetLastBuildFinishedAt()) + } + if input.GetLastCheckedIn() > 0 { // update LastCheckedIn if set w.SetLastCheckedIn(input.GetLastCheckedIn()) diff --git a/database/worker/create_test.go b/database/worker/create_test.go index 38c276d83..937e6564e 100644 --- a/database/worker/create_test.go +++ b/database/worker/create_test.go @@ -26,9 +26,9 @@ func TestWorker_Engine_CreateWorker(t *testing.T) { // ensure the mock expects the query _mock.ExpectQuery(`INSERT INTO "workers" -("hostname","address","routes","active","last_checked_in","build_limit","id") -VALUES ($1,$2,$3,$4,$5,$6,$7) RETURNING "id"`). - WithArgs("worker_0", "localhost", nil, true, nil, nil, 1). +("hostname","address","routes","active","status","last_status_update_at","running_build_ids","last_build_finished_at","last_checked_in","build_limit","id") +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING "id"`). + WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, 1). WillReturnRows(_rows) _sqlite := testSqlite(t) diff --git a/database/worker/table.go b/database/worker/table.go index 5cecf109d..55d0e3c52 100644 --- a/database/worker/table.go +++ b/database/worker/table.go @@ -14,29 +14,36 @@ const ( CREATE TABLE IF NOT EXISTS workers ( - id SERIAL PRIMARY KEY, - hostname VARCHAR(250), - address VARCHAR(250), - routes VARCHAR(1000), - active BOOLEAN, - last_checked_in INTEGER, - build_limit INTEGER, + id SERIAL PRIMARY KEY, + hostname VARCHAR(250), + address VARCHAR(250), + routes VARCHAR(1000), + active BOOLEAN, + status VARCHAR(50), + last_status_update_at INTEGER, + running_build_ids VARCHAR(1000), + last_build_finished_at INTEGER, + last_checked_in INTEGER, + build_limit INTEGER, UNIQUE(hostname) ); ` - // CreateSqliteTable represents a query to create the Sqlite workers table. CreateSqliteTable = ` CREATE TABLE IF NOT EXISTS workers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - hostname TEXT, - address TEXT, - routes TEXT, - active BOOLEAN, - last_checked_in INTEGER, - build_limit INTEGER, + id INTEGER PRIMARY KEY AUTOINCREMENT, + hostname TEXT, + address TEXT, + routes TEXT, + active BOOLEAN, + status VARCHAR(50), + last_status_update_at INTEGER, + running_build_ids VARCHAR(1000), + last_build_finished_at INTEGER, + last_checked_in INTEGER, + build_limit INTEGER, UNIQUE(hostname) ); ` diff --git a/database/worker/update_test.go b/database/worker/update_test.go index 88644678a..3e15cafa1 100644 --- a/database/worker/update_test.go +++ b/database/worker/update_test.go @@ -23,9 +23,9 @@ func TestWorker_Engine_UpdateWorker(t *testing.T) { // ensure the mock expects the query _mock.ExpectExec(`UPDATE "workers" -SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"last_checked_in"=$5,"build_limit"=$6 -WHERE "id" = $7`). - WithArgs("worker_0", "localhost", nil, true, nil, nil, 1). +SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"status"=$5,"last_status_update_at"=$6,"running_build_ids"=$7,"last_build_finished_at"=$8,"last_checked_in"=$9,"build_limit"=$10 +WHERE "id" = $11`). + WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/worker/worker_test.go b/database/worker/worker_test.go index 48deabe54..3e88cc26b 100644 --- a/database/worker/worker_test.go +++ b/database/worker/worker_test.go @@ -170,12 +170,16 @@ func testSqlite(t *testing.T) *engine { // Worker type with all fields set to their zero values. func testWorker() *library.Worker { return &library.Worker{ - ID: new(int64), - Hostname: new(string), - Address: new(string), - Routes: new([]string), - Active: new(bool), - BuildLimit: new(int64), - LastCheckedIn: new(int64), + ID: new(int64), + Hostname: new(string), + Address: new(string), + Routes: new([]string), + Active: new(bool), + Status: new(string), + LastStatusUpdateAt: new(int64), + RunningBuildIDs: new([]string), + LastBuildFinishedAt: new(int64), + LastCheckedIn: new(int64), + BuildLimit: new(int64), } } diff --git a/router/middleware/worker/worker_test.go b/router/middleware/worker/worker_test.go index cad19f15e..d65102189 100644 --- a/router/middleware/worker/worker_test.go +++ b/router/middleware/worker/worker_test.go @@ -42,6 +42,10 @@ func TestWorker_Establish(t *testing.T) { want.SetAddress("localhost") want.SetRoutes([]string{"foo", "bar", "baz"}) want.SetActive(true) + want.SetStatus("available") + want.SetLastStatusUpdateAt(12345) + want.SetRunningBuildIDs([]string{}) + want.SetLastBuildFinishedAt(12345) want.SetLastCheckedIn(12345) want.SetBuildLimit(0) From 822800a5e924bee07b70414a8c5da94750f255c3 Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Thu, 6 Apr 2023 13:45:28 -0500 Subject: [PATCH 02/14] fix(api-worker.go): clean up merge text --- api/worker.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/api/worker.go b/api/worker.go index b2fdbb59c..9c12b4ca7 100644 --- a/api/worker.go +++ b/api/worker.go @@ -309,9 +309,8 @@ func UpdateWorker(c *gin.Context) { if input.GetActive() { // update active if set w.SetActive(input.GetActive()) - } + ) -<<<<<<< HEAD if len(input.GetStatus()) > 0 { // update status if set w.SetStatus(input.GetStatus()) @@ -336,9 +335,7 @@ func UpdateWorker(c *gin.Context) { // update LastCheckedIn if set w.SetLastCheckedIn(input.GetLastCheckedIn()) } - -======= ->>>>>>> 34164d0412e2bf5f966829420c325180f851e9c1 + // send API call to update the worker err = database.FromContext(c).UpdateWorker(w) if err != nil { From 540d7d076868bb5f3a658f8bf4c780b671e9a011 Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Wed, 12 Apr 2023 09:45:15 -0500 Subject: [PATCH 03/14] fix(api/worker.go): correct closing brace --- api/worker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/worker.go b/api/worker.go index 85d1899e4..8428b9ab7 100644 --- a/api/worker.go +++ b/api/worker.go @@ -321,7 +321,7 @@ func UpdateWorker(c *gin.Context) { if input.GetActive() { // update active if set w.SetActive(input.GetActive()) - ) + } if len(input.GetStatus()) > 0 { // update status if set @@ -347,7 +347,7 @@ func UpdateWorker(c *gin.Context) { // update LastCheckedIn if set w.SetLastCheckedIn(input.GetLastCheckedIn()) } - + // send API call to update the worker err = database.FromContext(c).UpdateWorker(w) if err != nil { From b9ae9af72ad54d3ea48ba55026f588f67ceb9082 Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Thu, 18 May 2023 09:30:46 -0500 Subject: [PATCH 04/14] feat(metrics.go): add idle_worker_count --- api/metrics.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/metrics.go b/api/metrics.go index fe5899d3a..120e2295d 100644 --- a/api/metrics.go +++ b/api/metrics.go @@ -474,6 +474,11 @@ func recordGauges(c *gin.Context) { totals.WithLabelValues("worker", "count", "inactive").Set(float64(inactiveWorkers)) } + // idle_worker_count + if q.IdleWorkerCount { + totals.WithLabelValues("worker", "count", "idle").Set(float64(idleWorkers)) + } + // available_worker_count if q.AvailableWorkerCount { totals.WithLabelValues("worker", "count", "available").Set(float64(availableWorkers)) From 49ca50df4845a9f213a818b8a8e513bd7865bd24 Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Mon, 22 May 2023 14:37:03 -0500 Subject: [PATCH 05/14] feat(worker): modify permissions to all worker for work towards updating its status --- router/middleware/perm/perm.go | 9 +++++++++ router/worker.go | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/router/middleware/perm/perm.go b/router/middleware/perm/perm.go index 812c9a8fd..da4c18013 100644 --- a/router/middleware/perm/perm.go +++ b/router/middleware/perm/perm.go @@ -100,6 +100,15 @@ func MustWorkerAuthToken() gin.HandlerFunc { "worker": cl.Subject, }).Debugf("verifying worker %s has a valid auth token", cl.Subject) + // global permissions bypass + if cl.IsAdmin { + logrus.WithFields(logrus.Fields{ + "user": cl.Subject, + }).Debugf("user %s has platform admin permissions", cl.Subject) + + return + } + switch cl.TokenType { case constants.WorkerAuthTokenType, constants.WorkerRegisterTokenType: return diff --git a/router/worker.go b/router/worker.go index 7bb114c69..367e1388c 100644 --- a/router/worker.go +++ b/router/worker.go @@ -32,7 +32,7 @@ func WorkerHandlers(base *gin.RouterGroup) { _worker := _workers.Group("/:worker") { _worker.GET("", wmiddleware.Establish(), worker.GetWorker) - _worker.PUT("", perm.MustPlatformAdmin(), wmiddleware.Establish(), worker.UpdateWorker) + _worker.PUT("", perm.MustPlatformAdmin(), perm.MustWorkerAuthToken(), wmiddleware.Establish(), worker.UpdateWorker) _worker.POST("/refresh", perm.MustWorkerAuthToken(), wmiddleware.Establish(), worker.Refresh) _worker.DELETE("", perm.MustPlatformAdmin(), wmiddleware.Establish(), worker.DeleteWorker) } // end of worker endpoints From e5e7c08b8f9f1a71a0a1eb70fce05882f601ab6a Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Wed, 31 May 2023 13:21:23 -0500 Subject: [PATCH 06/14] fix(runningBuildIDs): allow nil update to list --- api/worker/update.go | 10 ++++++++++ router/worker.go | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/api/worker/update.go b/api/worker/update.go index 0c2f9c4fb..8580d9c24 100644 --- a/api/worker/update.go +++ b/api/worker/update.go @@ -98,6 +98,16 @@ func UpdateWorker(c *gin.Context) { w.SetActive(input.GetActive()) } + if input.GetRunningBuildIDs() != nil { + // update status if set + w.SetRunningBuildIDs(input.GetRunningBuildIDs()) + } + + if len(input.GetStatus()) > 0 { + // update status if set + w.SetStatus(input.GetStatus()) + } + // send API call to update the worker err = database.FromContext(c).UpdateWorker(w) if err != nil { diff --git a/router/worker.go b/router/worker.go index 367e1388c..f85a84c45 100644 --- a/router/worker.go +++ b/router/worker.go @@ -32,7 +32,7 @@ func WorkerHandlers(base *gin.RouterGroup) { _worker := _workers.Group("/:worker") { _worker.GET("", wmiddleware.Establish(), worker.GetWorker) - _worker.PUT("", perm.MustPlatformAdmin(), perm.MustWorkerAuthToken(), wmiddleware.Establish(), worker.UpdateWorker) + _worker.PUT("", perm.MustWorkerAuthToken(), wmiddleware.Establish(), worker.UpdateWorker) _worker.POST("/refresh", perm.MustWorkerAuthToken(), wmiddleware.Establish(), worker.Refresh) _worker.DELETE("", perm.MustPlatformAdmin(), wmiddleware.Establish(), worker.DeleteWorker) } // end of worker endpoints From e8eda6acbf440bad04ad5ba85323c14dc71936f2 Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Thu, 1 Jun 2023 09:15:13 -0500 Subject: [PATCH 07/14] feat(worker): update last_status_update_at in the database --- api/worker/update.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/worker/update.go b/api/worker/update.go index 8580d9c24..523805b5c 100644 --- a/api/worker/update.go +++ b/api/worker/update.go @@ -99,7 +99,7 @@ func UpdateWorker(c *gin.Context) { } if input.GetRunningBuildIDs() != nil { - // update status if set + // update runningBuildIDs if set w.SetRunningBuildIDs(input.GetRunningBuildIDs()) } @@ -108,6 +108,11 @@ func UpdateWorker(c *gin.Context) { w.SetStatus(input.GetStatus()) } + if input.GetLastStatusUpdateAt() > 0 { + // update lastStatusUpdateAt if set + w.SetLastStatusUpdateAt(input.GetLastStatusUpdateAt()) + } + // send API call to update the worker err = database.FromContext(c).UpdateWorker(w) if err != nil { From 4d79db4b6fe3df180a78b7ee81fd06c031e6beb3 Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Fri, 2 Jun 2023 11:08:12 -0500 Subject: [PATCH 08/14] feat(worker): add LastBuildStartedAt field --- api/worker.go | 5 +++++ database/worker/create_test.go | 4 ++-- database/worker/table.go | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/api/worker.go b/api/worker.go index 8428b9ab7..9ed77feb9 100644 --- a/api/worker.go +++ b/api/worker.go @@ -338,6 +338,11 @@ func UpdateWorker(c *gin.Context) { w.SetRunningBuildIDs(input.GetRunningBuildIDs()) } + if input.GetLastBuildStartedAt() > 0 { + // update LastBuildStartedAt if set + w.SetLastBuildStartedAt(input.GetLastBuildStartedAt()) + } + if input.GetLastBuildFinishedAt() > 0 { // update LastBuildFinishedAt if set w.SetLastBuildFinishedAt(input.GetLastBuildFinishedAt()) diff --git a/database/worker/create_test.go b/database/worker/create_test.go index 937e6564e..863d070f5 100644 --- a/database/worker/create_test.go +++ b/database/worker/create_test.go @@ -26,9 +26,9 @@ func TestWorker_Engine_CreateWorker(t *testing.T) { // ensure the mock expects the query _mock.ExpectQuery(`INSERT INTO "workers" -("hostname","address","routes","active","status","last_status_update_at","running_build_ids","last_build_finished_at","last_checked_in","build_limit","id") +("hostname","address","routes","active","status","last_status_update_at","running_build_ids","last_build_started_at","last_checked_in","build_limit","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING "id"`). - WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, 1). + WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, nil, 1). WillReturnRows(_rows) _sqlite := testSqlite(t) diff --git a/database/worker/table.go b/database/worker/table.go index 55d0e3c52..26409be21 100644 --- a/database/worker/table.go +++ b/database/worker/table.go @@ -22,6 +22,7 @@ workers ( status VARCHAR(50), last_status_update_at INTEGER, running_build_ids VARCHAR(1000), + last_build_started_at INTEGER, last_build_finished_at INTEGER, last_checked_in INTEGER, build_limit INTEGER, @@ -41,6 +42,7 @@ workers ( status VARCHAR(50), last_status_update_at INTEGER, running_build_ids VARCHAR(1000), + last_build_started_at INTEGER, last_build_finished_at INTEGER, last_checked_in INTEGER, build_limit INTEGER, From 6073481c8a05b3e64122d4313e92f7b33cc08e0d Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Fri, 2 Jun 2023 12:16:18 -0500 Subject: [PATCH 09/14] fix(worker): allow update of last build started and finished --- api/worker/update.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api/worker/update.go b/api/worker/update.go index 523805b5c..0b5014114 100644 --- a/api/worker/update.go +++ b/api/worker/update.go @@ -113,6 +113,16 @@ func UpdateWorker(c *gin.Context) { w.SetLastStatusUpdateAt(input.GetLastStatusUpdateAt()) } + if input.GetLastBuildStartedAt() > 0 { + // update lastBuildStartedAt if set + w.SetLastBuildStartedAt(input.GetLastBuildStartedAt()) + } + + if input.GetLastBuildFinishedAt() > 0 { + // update lastBuildFinishedAt if set + w.SetLastBuildFinishedAt(input.GetLastBuildFinishedAt()) + } + // send API call to update the worker err = database.FromContext(c).UpdateWorker(w) if err != nil { From 99541ac2c5e6d5344c3aa03305b4e2bdd09fef82 Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Thu, 8 Jun 2023 16:16:08 -0500 Subject: [PATCH 10/14] test(worker): ensure all new fields are added to relevant tests --- database/worker/create_test.go | 4 ++-- database/worker/get_hostname_test.go | 4 ++-- database/worker/index_test.go | 2 +- database/worker/list_test.go | 6 +++--- database/worker/update_test.go | 6 +++--- database/worker/worker_test.go | 1 + router/middleware/worker/worker_test.go | 1 + 7 files changed, 13 insertions(+), 11 deletions(-) diff --git a/database/worker/create_test.go b/database/worker/create_test.go index 863d070f5..e4c4dc9cb 100644 --- a/database/worker/create_test.go +++ b/database/worker/create_test.go @@ -26,8 +26,8 @@ func TestWorker_Engine_CreateWorker(t *testing.T) { // ensure the mock expects the query _mock.ExpectQuery(`INSERT INTO "workers" -("hostname","address","routes","active","status","last_status_update_at","running_build_ids","last_build_started_at","last_checked_in","build_limit","id") -VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING "id"`). +("hostname","address","routes","active","status","last_status_update_at","running_build_ids","last_build_started_at","last_build_finished_at","last_checked_in","build_limit","id") +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) RETURNING "id"`). WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, nil, 1). WillReturnRows(_rows) diff --git a/database/worker/get_hostname_test.go b/database/worker/get_hostname_test.go index 833afdc15..3dd1d4fe6 100644 --- a/database/worker/get_hostname_test.go +++ b/database/worker/get_hostname_test.go @@ -25,8 +25,8 @@ func TestWorker_Engine_GetWorkerForName(t *testing.T) { // create expected result in mock _rows := sqlmock.NewRows( - []string{"id", "hostname", "address", "routes", "active", "last_checked_in", "build_limit"}). - AddRow(1, "worker_0", "localhost", nil, true, 0, 0) + []string{"id", "hostname", "address", "routes", "active", "status", "last_status_update_at", "running_build_ids", "last_build_started_at", "last_build_finished_at", "last_checked_in", "build_limit"}). + AddRow(1, "worker_0", "localhost", nil, true, nil, 0, nil, 0, 0, 0, 0) // ensure the mock expects the query _mock.ExpectQuery(`SELECT * FROM "workers" WHERE hostname = $1 LIMIT 1`).WithArgs("worker_0").WillReturnRows(_rows) diff --git a/database/worker/index_test.go b/database/worker/index_test.go index ead204e5c..b7b06d347 100644 --- a/database/worker/index_test.go +++ b/database/worker/index_test.go @@ -10,7 +10,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" ) -func TestWorker_Engine_CreateWorkerIndexes(t *testing.T) { +func Indexes(t *testing.T) { // setup types _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() diff --git a/database/worker/list_test.go b/database/worker/list_test.go index b44c9c3d9..5eed3f94f 100644 --- a/database/worker/list_test.go +++ b/database/worker/list_test.go @@ -37,9 +37,9 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { // create expected result in mock _rows = sqlmock.NewRows( - []string{"id", "hostname", "address", "routes", "active", "last_checked_in", "build_limit"}). - AddRow(1, "worker_0", "localhost", nil, true, 0, 0). - AddRow(2, "worker_1", "localhost", nil, true, 0, 0) + []string{"id", "hostname", "address", "routes", "active", "status", "last_status_update_at", "running_build_ids", "last_build_started_at", "last_build_finished_at", "last_checked_in", "build_limit"}). + AddRow(1, "worker_0", "localhost", nil, true, nil, 0, nil, 0, 0, 0, 0). + AddRow(2, "worker_1", "localhost", nil, true, nil, 0, nil, 0, 0, 0, 0) // ensure the mock expects the query _mock.ExpectQuery(`SELECT * FROM "workers"`).WillReturnRows(_rows) diff --git a/database/worker/update_test.go b/database/worker/update_test.go index 3e15cafa1..0beeafa47 100644 --- a/database/worker/update_test.go +++ b/database/worker/update_test.go @@ -23,9 +23,9 @@ func TestWorker_Engine_UpdateWorker(t *testing.T) { // ensure the mock expects the query _mock.ExpectExec(`UPDATE "workers" -SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"status"=$5,"last_status_update_at"=$6,"running_build_ids"=$7,"last_build_finished_at"=$8,"last_checked_in"=$9,"build_limit"=$10 -WHERE "id" = $11`). - WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, 1). +SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"status"=$5,"last_status_update_at"=$6,"running_build_ids"=$7,"last_build_started_at"=$8,"last_build_finished_at"=$9,"last_checked_in"=$10,"build_limit"=$11 +WHERE "id" = $12`). + WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, nil, 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/worker/worker_test.go b/database/worker/worker_test.go index 3e88cc26b..00096c2a9 100644 --- a/database/worker/worker_test.go +++ b/database/worker/worker_test.go @@ -178,6 +178,7 @@ func testWorker() *library.Worker { Status: new(string), LastStatusUpdateAt: new(int64), RunningBuildIDs: new([]string), + LastBuildStartedAt: new(int64), LastBuildFinishedAt: new(int64), LastCheckedIn: new(int64), BuildLimit: new(int64), diff --git a/router/middleware/worker/worker_test.go b/router/middleware/worker/worker_test.go index d65102189..7b77c7f3b 100644 --- a/router/middleware/worker/worker_test.go +++ b/router/middleware/worker/worker_test.go @@ -45,6 +45,7 @@ func TestWorker_Establish(t *testing.T) { want.SetStatus("available") want.SetLastStatusUpdateAt(12345) want.SetRunningBuildIDs([]string{}) + want.SetLastBuildStartedAt(12345) want.SetLastBuildFinishedAt(12345) want.SetLastCheckedIn(12345) want.SetBuildLimit(0) From a79873a0b6d340c60c90ba3745defa1be194d645 Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Mon, 12 Jun 2023 13:52:09 -0500 Subject: [PATCH 11/14] chore(worker_visibility): pull in types and server main --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index be974371f..ee0452134 100644 --- a/go.mod +++ b/go.mod @@ -7,14 +7,14 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 - github.com/adhocore/gronx v1.6.2 + github.com/adhocore/gronx v1.6.3 github.com/alicebob/miniredis/v2 v2.30.2 github.com/aws/aws-sdk-go v1.44.248 github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.0 github.com/go-playground/assert/v2 v2.2.0 - github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c + github.com/go-vela/types v0.19.3-0.20230612142147-a13b7ca8ce2c github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/go-cmp v0.5.9 github.com/google/go-github/v52 v52.0.0 @@ -87,12 +87,12 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/leodido/go-urn v1.2.1 // indirect - github.com/lib/pq v1.10.8 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/microcosm-cc/bluemonday v1.0.23 // indirect + github.com/microcosm-cc/bluemonday v1.0.24 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -114,8 +114,8 @@ require ( github.com/yuin/gopher-lua v1.1.0 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect golang.org/x/crypto v0.7.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 3b4b80685..be952bf6a 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,8 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/adhocore/gronx v1.6.2 h1:/Pg6cuHFJmUGRIYWhRFjb6iL9fdzNmoMPj+/r6L01KU= -github.com/adhocore/gronx v1.6.2/go.mod h1:7oUY1WAU8rEJWmAxXR2DN0JaO4gi9khSgKjiRypqteg= +github.com/adhocore/gronx v1.6.3 h1:bnm5vieTrY3QQPpsfB0hrAaeaHDpuZTUC2LLCVMLe9c= +github.com/adhocore/gronx v1.6.3/go.mod h1:7oUY1WAU8rEJWmAxXR2DN0JaO4gi9khSgKjiRypqteg= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= @@ -138,8 +138,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= -github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c h1:eAApIK5e5MxFF8RzZAFsvTSdwq/AzdUrdhJHOGQ0ILc= -github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c/go.mod h1:0lsuPfGyVyTWJSi2h3NS6uaEW6DgnFvIzaZu1sXYKrs= +github.com/go-vela/types v0.19.3-0.20230612142147-a13b7ca8ce2c h1:w/XLwm4Y2puSTU5akAFKx5LI8BCPt+e0wofrTEsyDQc= +github.com/go-vela/types v0.19.3-0.20230612142147-a13b7ca8ce2c/go.mod h1:1ZSmKWX9MamKogwaIb53mzzRpZMV34mJFKiGfVFadFk= github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -293,8 +293,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lib/pq v1.10.8 h1:3fdt97i/cwSU83+E0hZTC/Xpc9mTZxc6UWSCRcSbxiE= -github.com/lib/pq v1.10.8/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -309,8 +309,8 @@ github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJK github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY= -github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4= +github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw= +github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= @@ -486,8 +486,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -558,8 +558,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From c648ab81fa2eef70b25137ea7883ef264ad6199e Mon Sep 17 00:00:00 2001 From: TimHuynh Date: Wed, 14 Jun 2023 14:35:25 -0500 Subject: [PATCH 12/14] addressed feedback --- api/metrics.go | 5 +- api/worker.go | 539 ---------------------------------- database/worker/index_test.go | 2 +- 3 files changed, 2 insertions(+), 544 deletions(-) delete mode 100644 api/worker.go diff --git a/api/metrics.go b/api/metrics.go index d80d7f8da..fc1a7f04f 100644 --- a/api/metrics.go +++ b/api/metrics.go @@ -430,6 +430,7 @@ func recordGauges(c *gin.Context) { before := time.Now().UTC().Add(-c.Value("worker_active_interval").(time.Duration)).Unix() // active, inactive counts + // idle, available, busy, error counts for _, worker := range workers { // check if the worker checked in within the last worker_active_interval if worker.GetLastCheckedIn() >= before { @@ -438,10 +439,6 @@ func recordGauges(c *gin.Context) { } else { inactiveWorkers++ } - } - - // idle, available, busy, error counts - for _, worker := range workers { // check if the worker checked in within the last worker_active_interval if worker.GetLastCheckedIn() >= before { diff --git a/api/worker.go b/api/worker.go deleted file mode 100644 index 9ed77feb9..000000000 --- a/api/worker.go +++ /dev/null @@ -1,539 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package api - -import ( - "fmt" - "net/http" - "strings" - "time" - - "github.com/go-vela/server/internal/token" - "github.com/go-vela/server/router/middleware/claims" - "github.com/go-vela/server/router/middleware/user" - "github.com/go-vela/types/constants" - - "github.com/go-vela/server/database" - "github.com/go-vela/server/router/middleware/worker" - "github.com/go-vela/server/util" - - "github.com/go-vela/types/library" - - "github.com/gin-gonic/gin" - "github.com/sirupsen/logrus" -) - -// swagger:operation POST /api/v1/workers workers CreateWorker -// -// Create a worker for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: body -// name: body -// description: Payload containing the worker to create -// required: true -// schema: -// "$ref": "#/definitions/Worker" -// security: -// - ApiKeyAuth: [] -// responses: -// '201': -// description: Successfully created the worker and retrieved auth token -// schema: -// "$ref": "#definitions/Token" -// '400': -// description: Unable to create the worker -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to create the worker -// schema: -// "$ref": "#/definitions/Error" - -// CreateWorker represents the API handler to -// create a worker in the configured backend. -func CreateWorker(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - cl := claims.Retrieve(c) - - // capture body from API request - input := new(library.Worker) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for new worker: %w", err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // verify input host name matches worker hostname - if !strings.EqualFold(cl.TokenType, constants.ServerWorkerTokenType) && !strings.EqualFold(cl.Subject, input.GetHostname()) { - retErr := fmt.Errorf("unable to add worker; claims subject %s does not match worker hostname %s", cl.Subject, input.GetHostname()) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - input.SetLastCheckedIn(time.Now().Unix()) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "user": u.GetName(), - "worker": input.GetHostname(), - }).Infof("creating new worker %s", input.GetHostname()) - - err = database.FromContext(c).CreateWorker(input) - if err != nil { - retErr := fmt.Errorf("unable to create worker: %w", err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - switch cl.TokenType { - // if symmetric token configured, send back symmetric token - case constants.ServerWorkerTokenType: - if secret, ok := c.Value("secret").(string); ok { - tkn := new(library.Token) - tkn.SetToken(secret) - c.JSON(http.StatusCreated, tkn) - - return - } - - retErr := fmt.Errorf("symmetric token provided but not configured in server") - util.HandleError(c, http.StatusBadRequest, retErr) - - return - // if worker register token, send back auth token - default: - tm := c.MustGet("token-manager").(*token.Manager) - - wmto := &token.MintTokenOpts{ - TokenType: constants.WorkerAuthTokenType, - TokenDuration: tm.WorkerAuthTokenDuration, - Hostname: cl.Subject, - } - - tkn := new(library.Token) - - wt, err := tm.MintToken(wmto) - if err != nil { - retErr := fmt.Errorf("unable to generate auth token for worker %s: %w", input.GetHostname(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - tkn.SetToken(wt) - - c.JSON(http.StatusCreated, tkn) - } -} - -// swagger:operation GET /api/v1/workers workers GetWorkers -// -// Retrieve a list of workers for the configured backend -// -// --- -// produces: -// - application/json -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the list of workers -// schema: -// type: array -// items: -// "$ref": "#/definitions/Worker" -// '500': -// description: Unable to retrieve the list of workers -// schema: -// "$ref": "#/definitions/Error" - -// GetWorkers represents the API handler to capture a -// list of workers from the configured backend. -func GetWorkers(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "user": u.GetName(), - }).Info("reading workers") - - w, err := database.FromContext(c).ListWorkers() - if err != nil { - retErr := fmt.Errorf("unable to get workers: %w", err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, w) -} - -// swagger:operation GET /api/v1/workers/{worker} workers GetWorker -// -// Retrieve a worker for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: worker -// description: Hostname of the worker -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully retrieved the worker -// schema: -// "$ref": "#/definitions/Worker" -// '404': -// description: Unable to retrieve the worker -// schema: -// "$ref": "#/definitions/Error" - -// GetWorker represents the API handler to capture a -// worker from the configured backend. -func GetWorker(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - w := worker.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "user": u.GetName(), - "worker": w.GetHostname(), - }).Infof("reading worker %s", w.GetHostname()) - - w, err := database.FromContext(c).GetWorkerForHostname(w.GetHostname()) - if err != nil { - retErr := fmt.Errorf("unable to get workers: %w", err) - - util.HandleError(c, http.StatusNotFound, retErr) - - return - } - - c.JSON(http.StatusOK, w) -} - -// swagger:operation PUT /api/v1/workers/{worker} workers UpdateWorker -// -// Update a worker for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: body -// name: body -// description: Payload containing the worker to update -// required: true -// schema: -// "$ref": "#/definitions/Worker" -// - in: path -// name: worker -// description: Name of the worker -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully updated the worker -// schema: -// "$ref": "#/definitions/Worker" -// '400': -// description: Unable to update the worker -// schema: -// "$ref": "#/definitions/Error" -// '404': -// description: Unable to update the worker -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to update the worker -// schema: -// "$ref": "#/definitions/Error" - -// UpdateWorker represents the API handler to -// update a worker in the configured backend. -func UpdateWorker(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - w := worker.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "user": u.GetName(), - "worker": w.GetHostname(), - }).Infof("updating worker %s", w.GetHostname()) - - // capture body from API request - input := new(library.Worker) - - err := c.Bind(input) - if err != nil { - retErr := fmt.Errorf("unable to decode JSON for worker %s: %w", w.GetHostname(), err) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - if len(input.GetAddress()) > 0 { - // update admin if set - w.SetAddress(input.GetAddress()) - } - - if len(input.GetRoutes()) > 0 { - // update routes if set - w.SetRoutes(input.GetRoutes()) - } - - if input.GetActive() { - // update active if set - w.SetActive(input.GetActive()) - } - - if len(input.GetStatus()) > 0 { - // update status if set - w.SetStatus(input.GetStatus()) - } - - if input.GetLastStatusUpdateAt() > 0 { - // update LastStatusUpdateAt if set - w.SetLastStatusUpdateAt(input.GetLastStatusUpdateAt()) - } - - if len(input.GetRunningBuildIDs()) > 0 { - // update RunningBuildIDs if set - w.SetRunningBuildIDs(input.GetRunningBuildIDs()) - } - - if input.GetLastBuildStartedAt() > 0 { - // update LastBuildStartedAt if set - w.SetLastBuildStartedAt(input.GetLastBuildStartedAt()) - } - - if input.GetLastBuildFinishedAt() > 0 { - // update LastBuildFinishedAt if set - w.SetLastBuildFinishedAt(input.GetLastBuildFinishedAt()) - } - - if input.GetLastCheckedIn() > 0 { - // update LastCheckedIn if set - w.SetLastCheckedIn(input.GetLastCheckedIn()) - } - - // send API call to update the worker - err = database.FromContext(c).UpdateWorker(w) - if err != nil { - retErr := fmt.Errorf("unable to update worker %s: %w", w.GetHostname(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // send API call to capture the updated worker - w, _ = database.FromContext(c).GetWorkerForHostname(w.GetHostname()) - - c.JSON(http.StatusOK, w) -} - -// swagger:operation POST /api/v1/workers/{worker}/refresh workers RefreshWorkerAuth -// -// Refresh authorization token for worker -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: worker -// description: Name of the worker -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully refreshed auth -// schema: -// "$ref": "#/definitions/Token" -// '400': -// description: Unable to refresh worker auth -// schema: -// "$ref": "#/definitions/Error" -// '404': -// description: Unable to refresh worker auth -// schema: -// "$ref": "#/definitions/Error" -// '500': -// description: Unable to refresh worker auth -// schema: -// "$ref": "#/definitions/Error" - -// RefreshWorkerAuth represents the API handler to -// refresh the auth token for a worker. -func RefreshWorkerAuth(c *gin.Context) { - // capture middleware values - w := worker.Retrieve(c) - cl := claims.Retrieve(c) - - // if we are not using a symmetric token, and the subject does not match the input, request should be denied - if !strings.EqualFold(cl.TokenType, constants.ServerWorkerTokenType) && !strings.EqualFold(cl.Subject, w.GetHostname()) { - retErr := fmt.Errorf("unable to refresh worker auth: claims subject %s does not match worker hostname %s", cl.Subject, w.GetHostname()) - - logrus.WithFields(logrus.Fields{ - "subject": cl.Subject, - "worker": w.GetHostname(), - }).Warnf("attempted refresh of worker %s using token from worker %s", w.GetHostname(), cl.Subject) - - util.HandleError(c, http.StatusBadRequest, retErr) - - return - } - - // set last checked in time - w.SetLastCheckedIn(time.Now().Unix()) - - // send API call to update the worker - err := database.FromContext(c).UpdateWorker(w) - if err != nil { - retErr := fmt.Errorf("unable to update worker %s: %w", w.GetHostname(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "worker": w.GetHostname(), - }).Infof("refreshing worker %s authentication", w.GetHostname()) - - switch cl.TokenType { - // if symmetric token configured, send back symmetric token - case constants.ServerWorkerTokenType: - if secret, ok := c.Value("secret").(string); ok { - tkn := new(library.Token) - tkn.SetToken(secret) - c.JSON(http.StatusOK, tkn) - - return - } - - retErr := fmt.Errorf("symmetric token provided but not configured in server") - util.HandleError(c, http.StatusBadRequest, retErr) - - return - // if worker auth / register token, send back auth token - case constants.WorkerAuthTokenType, constants.WorkerRegisterTokenType: - tm := c.MustGet("token-manager").(*token.Manager) - - wmto := &token.MintTokenOpts{ - TokenType: constants.WorkerAuthTokenType, - TokenDuration: tm.WorkerAuthTokenDuration, - Hostname: cl.Subject, - } - - tkn := new(library.Token) - - wt, err := tm.MintToken(wmto) - if err != nil { - retErr := fmt.Errorf("unable to generate auth token for worker %s: %w", w.GetHostname(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - tkn.SetToken(wt) - - c.JSON(http.StatusOK, tkn) - } -} - -// swagger:operation DELETE /api/v1/workers/{worker} workers DeleteWorker -// -// Delete a worker for the configured backend -// -// --- -// produces: -// - application/json -// parameters: -// - in: path -// name: worker -// description: Name of the worker -// required: true -// type: string -// security: -// - ApiKeyAuth: [] -// responses: -// '200': -// description: Successfully deleted of worker -// schema: -// type: string -// '500': -// description: Unable to delete worker -// schema: -// "$ref": "#/definitions/Error" - -// DeleteWorker represents the API handler to remove -// a worker from the configured backend. -func DeleteWorker(c *gin.Context) { - // capture middleware values - u := user.Retrieve(c) - w := worker.Retrieve(c) - - // update engine logger with API metadata - // - // https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields - logrus.WithFields(logrus.Fields{ - "user": u.GetName(), - "worker": w.GetHostname(), - }).Infof("deleting worker %s", w.GetHostname()) - - // send API call to remove the step - err := database.FromContext(c).DeleteWorker(w) - if err != nil { - retErr := fmt.Errorf("unable to delete worker %s: %w", w.GetHostname(), err) - - util.HandleError(c, http.StatusInternalServerError, retErr) - - return - } - - c.JSON(http.StatusOK, fmt.Sprintf("worker %s deleted", w.GetHostname())) -} diff --git a/database/worker/index_test.go b/database/worker/index_test.go index b7b06d347..ead204e5c 100644 --- a/database/worker/index_test.go +++ b/database/worker/index_test.go @@ -10,7 +10,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" ) -func Indexes(t *testing.T) { +func TestWorker_Engine_CreateWorkerIndexes(t *testing.T) { // setup types _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() From 999a1db46b35bc600824637e22e61ca48b25ddfd Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Thu, 15 Jun 2023 10:06:19 -0500 Subject: [PATCH 13/14] fix(running_build_ids): correct copy/paste, size should be 500 --- database/worker/table.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/worker/table.go b/database/worker/table.go index 26409be21..1d704674a 100644 --- a/database/worker/table.go +++ b/database/worker/table.go @@ -21,7 +21,7 @@ workers ( active BOOLEAN, status VARCHAR(50), last_status_update_at INTEGER, - running_build_ids VARCHAR(1000), + running_build_ids VARCHAR(500), last_build_started_at INTEGER, last_build_finished_at INTEGER, last_checked_in INTEGER, @@ -41,7 +41,7 @@ workers ( active BOOLEAN, status VARCHAR(50), last_status_update_at INTEGER, - running_build_ids VARCHAR(1000), + running_build_ids VARCHAR(500), last_build_started_at INTEGER, last_build_finished_at INTEGER, last_checked_in INTEGER, From 56aea856ab2fc237085df6cc6d7f706debda0ddf Mon Sep 17 00:00:00 2001 From: Kelly Merrick Date: Thu, 15 Jun 2023 14:39:10 -0500 Subject: [PATCH 14/14] fix(running-builds): not requiring a list in body for updates --- api/worker/update.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/worker/update.go b/api/worker/update.go index cab888da3..b3a8d5130 100644 --- a/api/worker/update.go +++ b/api/worker/update.go @@ -98,7 +98,7 @@ func UpdateWorker(c *gin.Context) { w.SetActive(input.GetActive()) } - if input.GetRunningBuildIDs() != nil { + if input.RunningBuildIDs != nil { // update runningBuildIDs if set w.SetRunningBuildIDs(input.GetRunningBuildIDs()) }