feat(worker-visibility): extend worker table with 5 fields

api/metrics.go

@@ -11,6 +11,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/go-vela/server/database"
 	"github.com/go-vela/server/queue"
+	"github.com/go-vela/types/constants"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -56,6 +57,19 @@ type MetricsQueryParameters struct {
 	ActiveWorkerCount bool `form:"active_worker_count"`
 	// InactiveWorkerCount represents total number of inactive workers
 	InactiveWorkerCount bool `form:"inactive_worker_count"`
+
+	// UnregisteredWorkerCount represents total number of workers with a status of unregistered
+	UnregisteredWorkerCount bool `form:"unregistered_worker_count"`
+	// AvailableWorkerCount represents total number of workers with a status of available,
+	// where worker RunningBuildIDs.length < worker BuildLimit
+	AvailableWorkerCount bool `form:"available_worker_count"`
+	// BusyWorkerCount represents total number of workers with a status of busy,
+	// where worker BuildLimit == worker RunningBuildIDs.length
+	BusyWorkerCount bool `form:"busy_worker_count"`
+	// BusyWorkerCount represents total number of workers with a status of maintenance
+	MaintenanceWorkerCount bool `form:"maintenance_worker_count"`
+	// ErrorWorkerCount represents total number of workers with a status of error
+	ErrorWorkerCount bool `form:"error_worker_count"`
 }
 
 // predefine Prometheus metrics else they will be regenerated
@@ -175,6 +189,31 @@ var (
 //   description: Indicates a request for inactive worker count
 //   type: boolean
 //   default: false
+// - in: query
+//   name: unregistered_worker_count
+//   description: Indicates a request for unregistered worker count
+//   type: boolean
+//   default: false
+// - in: query
+//   name: available_worker_count
+//   description: Indicates a request for available worker count
+//   type: boolean
+//   default: false
+// - in: query
+//   name: busy_worker_count
+//   description: Indicates a request for busy worker count
+//   type: boolean
+//   default: false
+// - in: query
+//   name: maintenance_worker_count
+//   description: Indicates a request for maintenance worker count
+//   type: boolean
+//   default: false
+// - in: query
+//   name: error_worker_count
+//   description: Indicates a request for error worker count
+//   type: boolean
+//   default: false
 // responses:
 //   '200':
 //     description: Successfully retrieved the Vela metrics
@@ -370,14 +409,19 @@ func recordGauges(c *gin.Context) {
 
 	// add worker metrics
 	var (
-		buildLimit      int64
-		activeWorkers   int64
-		inactiveWorkers int64
+		buildLimit          int64
+		activeWorkers       int64
+		inactiveWorkers     int64
+		unregisteredWorkers int64
+		availableWorkers    int64
+		busyWorkers         int64
+		maintenanceWorkers  int64
+		errorWorkers        int64
 	)
 
 	// get worker metrics based on request query parameters
-	// worker_build_limit, active_worker_count, inactive_worker_count
-	if q.WorkerBuildLimit || q.ActiveWorkerCount || q.InactiveWorkerCount {
+	// worker_build_limit, active_worker_count, inactive_worker_count, unregistered_worker_count, available_worker_count, busy_worker_count, maintenance_worker_count, error_worker_count
+	if q.WorkerBuildLimit || q.ActiveWorkerCount || q.InactiveWorkerCount || q.UnregisteredWorkerCount || q.AvailableWorkerCount || q.BusyWorkerCount || q.MaintenanceWorkerCount || q.ErrorWorkerCount {
 		// send API call to capture the workers
 		workers, err := database.FromContext(c).ListWorkers()
 		if err != nil {
@@ -386,6 +430,8 @@ func recordGauges(c *gin.Context) {
 
 		// get the unix time from worker_active_interval ago
 		before := time.Now().UTC().Add(-c.Value("worker_active_interval").(time.Duration)).Unix()
+
+		// active, inactive counts
 		for _, worker := range workers {
 			// check if the worker checked in within the last worker_active_interval
 			if worker.GetLastCheckedIn() >= before {
@@ -396,6 +442,31 @@ func recordGauges(c *gin.Context) {
 			}
 		}
 
+		// available, busy, maintenance, error counts
+		for _, worker := range workers {
+			// check if the worker checked in within the last worker_active_interval
+			if worker.GetLastCheckedIn() >= before {
+
+				switch worker.GetStatus() {
+				case constants.WorkerStatusUnregistered:
+					availableWorkers++
+				case constants.WorkerStatusBusy:
+					busyWorkers++
+				case constants.WorkerStatusMaintenance:
+					maintenanceWorkers++
+				case constants.WorkerStatusError:
+					errorWorkers++
+				}
+			}
+		}
+
+		// unregistered count
+		for _, worker := range workers {
+			if worker.GetStatus() == constants.WorkerStatusUnregistered {
+				unregisteredWorkers++
+			}
+		}
+
 		// apply metrics based on request query parameters
 		// worker_build_limit
 		if q.WorkerBuildLimit {
@@ -411,5 +482,30 @@ func recordGauges(c *gin.Context) {
 		if q.InactiveWorkerCount {
 			totals.WithLabelValues("worker", "count", "inactive").Set(float64(inactiveWorkers))
 		}
+
+		// unregistered_worker_count
+		if q.UnregisteredWorkerCount {
+			totals.WithLabelValues("worker", "count", "unregistered").Set(float64(unregisteredWorkers))
+		}
+
+		// available_worker_count
+		if q.AvailableWorkerCount {
+			totals.WithLabelValues("worker", "count", "available").Set(float64(availableWorkers))
+		}
+
+		// busy_worker_count
+		if q.BusyWorkerCount {
+			totals.WithLabelValues("worker", "count", "busy").Set(float64(busyWorkers))
+		}
+
+		// maintenance_worker_count
+		if q.MaintenanceWorkerCount {
+			totals.WithLabelValues("worker", "count", "maintenance").Set(float64(maintenanceWorkers))
+		}
+
+		// error_worker_count
+		if q.ErrorWorkerCount {
+			totals.WithLabelValues("worker", "count", "error").Set(float64(errorWorkers))
+		}
 	}
 }

api/worker.go

@@ -323,6 +323,31 @@ func UpdateWorker(c *gin.Context) {
 		w.SetActive(input.GetActive())
 	}
 
+	if len(input.GetStatus()) > 0 {
+		// update status if set
+		w.SetStatus(input.GetStatus())
+	}
+
+	if input.GetLastStatusUpdateAt() > 0 {
+		// update LastStatusUpdateAt if set
+		w.SetLastStatusUpdateAt(input.GetLastStatusUpdateAt())
+	}
+
+	if len(input.GetRunningBuildIDs()) > 0 {
+		// update RunningBuildIDs if set
+		w.SetRunningBuildIDs(input.GetRunningBuildIDs())
+	}
+
+	if input.GetLastBuildFinishedAt() > 0 {
+		// update LastBuildFinishedAt if set
+		w.SetLastBuildFinishedAt(input.GetLastBuildFinishedAt())
+	}
+
+	if input.GetLastCheckedIn() > 0 {
+		// update LastCheckedIn if set
+		w.SetLastCheckedIn(input.GetLastCheckedIn())
+	}
+
 	// send API call to update the worker
 	err = database.FromContext(c).UpdateWorker(w)
 	if err != nil {

database/worker/create_test.go

@@ -26,9 +26,9 @@ func TestWorker_Engine_CreateWorker(t *testing.T) {
 
 	// ensure the mock expects the query
 	_mock.ExpectQuery(`INSERT INTO "workers"
-("hostname","address","routes","active","last_checked_in","build_limit","id")
-VALUES ($1,$2,$3,$4,$5,$6,$7) RETURNING "id"`).
-		WithArgs("worker_0", "localhost", nil, true, nil, nil, 1).
+("hostname","address","routes","active","status","last_status_update_at","running_build_ids","last_build_finished_at","last_checked_in","build_limit","id")
+VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING "id"`).
+		WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, 1).
 		WillReturnRows(_rows)
 
 	_sqlite := testSqlite(t)

database/worker/table.go

@@ -14,29 +14,36 @@ const (
 CREATE TABLE
 IF NOT EXISTS
 workers (
-	id               SERIAL PRIMARY KEY,
-	hostname         VARCHAR(250),
-	address          VARCHAR(250),
-	routes           VARCHAR(1000),
-	active           BOOLEAN,
-	last_checked_in  INTEGER,
-	build_limit      INTEGER,
+	id                     SERIAL PRIMARY KEY,
+	hostname               VARCHAR(250),
+	address                VARCHAR(250),
+	routes                 VARCHAR(1000),
+	active                 BOOLEAN,
+	status                 VARCHAR(50),
+	last_status_update_at  INTEGER,
+	running_build_ids      VARCHAR(1000),
+	last_build_finished_at INTEGER,
+	last_checked_in        INTEGER,
+	build_limit            INTEGER,
 	UNIQUE(hostname)
 );
 `
-
 	// CreateSqliteTable represents a query to create the Sqlite workers table.
 	CreateSqliteTable = `
 CREATE TABLE
 IF NOT EXISTS
 workers (
-	id              INTEGER PRIMARY KEY AUTOINCREMENT,
-	hostname        TEXT,
-	address         TEXT,
-	routes          TEXT,
-	active          BOOLEAN,
-	last_checked_in	INTEGER,
-	build_limit     INTEGER,
+	id                     INTEGER PRIMARY KEY AUTOINCREMENT,
+	hostname               TEXT,
+	address                TEXT,
+	routes                 TEXT,
+	active                 BOOLEAN,
+	status                 VARCHAR(50),
+	last_status_update_at  INTEGER,
+	running_build_ids      VARCHAR(1000),
+	last_build_finished_at INTEGER,
+	last_checked_in	       INTEGER,
+	build_limit            INTEGER,
 	UNIQUE(hostname)
 );
 `

database/worker/update_test.go

@@ -23,9 +23,9 @@ func TestWorker_Engine_UpdateWorker(t *testing.T) {
 
 	// ensure the mock expects the query
 	_mock.ExpectExec(`UPDATE "workers"
-SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"last_checked_in"=$5,"build_limit"=$6
-WHERE "id" = $7`).
-		WithArgs("worker_0", "localhost", nil, true, nil, nil, 1).
+SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"status"=$5,"last_status_update_at"=$6,"running_build_ids"=$7,"last_build_finished_at"=$8,"last_checked_in"=$9,"build_limit"=$10
+WHERE "id" = $11`).
+		WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, 1).
 		WillReturnResult(sqlmock.NewResult(1, 1))
 
 	_sqlite := testSqlite(t)

database/worker/worker_test.go

@@ -170,12 +170,16 @@ func testSqlite(t *testing.T) *engine {
 // Worker type with all fields set to their zero values.
 func testWorker() *library.Worker {
 	return &library.Worker{
-		ID:            new(int64),
-		Hostname:      new(string),
-		Address:       new(string),
-		Routes:        new([]string),
-		Active:        new(bool),
-		BuildLimit:    new(int64),
-		LastCheckedIn: new(int64),
+		ID:                  new(int64),
+		Hostname:            new(string),
+		Address:             new(string),
+		Routes:              new([]string),
+		Active:              new(bool),
+		Status:              new(string),
+		LastStatusUpdateAt:  new(int64),
+		RunningBuildIDs:     new([]string),
+		LastBuildFinishedAt: new(int64),
+		LastCheckedIn:       new(int64),
+		BuildLimit:          new(int64),
 	}
 }

router/middleware/worker/worker_test.go

@@ -42,6 +42,10 @@ func TestWorker_Establish(t *testing.T) {
 	want.SetAddress("localhost")
 	want.SetRoutes([]string{"foo", "bar", "baz"})
 	want.SetActive(true)
+	want.SetStatus("available")
+	want.SetLastStatusUpdateAt(12345)
+	want.SetRunningBuildIDs([]string{})
+	want.SetLastBuildFinishedAt(12345)
 	want.SetLastCheckedIn(12345)
 	want.SetBuildLimit(0)