diff --git a/api/build.go b/api/build.go index 6abb48fee..4e35b7d33 100644 --- a/api/build.go +++ b/api/build.go @@ -1638,7 +1638,7 @@ func CancelBuild(c *gin.Context) { } // retrieve the worker info - w, err := database.FromContext(c).GetWorker(b.GetHost()) + w, err := database.FromContext(c).GetWorkerForHostname(b.GetHost()) if err != nil { retErr := fmt.Errorf("unable to get worker for build %s: %w", entry, err) util.HandleError(c, http.StatusNotFound, retErr) diff --git a/api/metrics.go b/api/metrics.go index edbe8ecdd..b5f6fe942 100644 --- a/api/metrics.go +++ b/api/metrics.go @@ -365,7 +365,7 @@ func recordGauges(c *gin.Context) { // worker_build_limit, active_worker_count, inactive_worker_count if q.WorkerBuildLimit || q.ActiveWorkerCount || q.InactiveWorkerCount { // send API call to capture the workers - workers, err := database.FromContext(c).GetWorkerList() + workers, err := database.FromContext(c).ListWorkers() if err != nil { logrus.Errorf("unable to get workers: %v", err) } diff --git a/api/worker.go b/api/worker.go index 6f5b65636..4951b182a 100644 --- a/api/worker.go +++ b/api/worker.go @@ -122,7 +122,7 @@ func GetWorkers(c *gin.Context) { "user": u.GetName(), }).Info("reading workers") - w, err := database.FromContext(c).GetWorkerList() + w, err := database.FromContext(c).ListWorkers() if err != nil { retErr := fmt.Errorf("unable to get workers: %w", err) @@ -174,7 +174,7 @@ func GetWorker(c *gin.Context) { "worker": w.GetHostname(), }).Infof("reading worker %s", w.GetHostname()) - w, err := database.FromContext(c).GetWorker(w.GetHostname()) + w, err := database.FromContext(c).GetWorkerForHostname(w.GetHostname()) if err != nil { retErr := fmt.Errorf("unable to get workers: %w", err) @@ -283,7 +283,7 @@ func UpdateWorker(c *gin.Context) { } // send API call to capture the updated worker - w, _ = database.FromContext(c).GetWorker(w.GetHostname()) + w, _ = database.FromContext(c).GetWorkerForHostname(w.GetHostname()) c.JSON(http.StatusOK, w) } @@ -329,7 +329,7 @@ func DeleteWorker(c *gin.Context) { }).Infof("deleting worker %s", w.GetHostname()) // send API call to remove the step - err := database.FromContext(c).DeleteWorker(w.GetID()) + err := database.FromContext(c).DeleteWorker(w) if err != nil { retErr := fmt.Errorf("unable to delete worker %s: %w", w.GetHostname(), err) diff --git a/database/postgres/ddl/worker.go b/database/postgres/ddl/worker.go deleted file mode 100644 index dd8658e7c..000000000 --- a/database/postgres/ddl/worker.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package ddl - -const ( - // CreateWorkerTable represents a query to - // create the workers table for Vela. - CreateWorkerTable = ` -CREATE TABLE -IF NOT EXISTS -workers ( - id SERIAL PRIMARY KEY, - hostname VARCHAR(250), - address VARCHAR(250), - routes VARCHAR(1000), - active BOOLEAN, - last_checked_in INTEGER, - build_limit INTEGER, - UNIQUE(hostname) -); -` - - // CreateWorkerHostnameAddressIndex represents a query to create an - // index on the workers table for the hostname and address columns. - CreateWorkerHostnameAddressIndex = ` -CREATE INDEX -IF NOT EXISTS -workers_hostname_address -ON workers (hostname, address); -` -) diff --git a/database/postgres/dml/worker.go b/database/postgres/dml/worker.go deleted file mode 100644 index 3ac74fd2e..000000000 --- a/database/postgres/dml/worker.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package dml - -const ( - // ListWorkers represents a query to - // list all workers in the database. - ListWorkers = ` -SELECT * -FROM workers; -` - - // SelectWorkersCount represents a query to select the - // count of workers in the database. - SelectWorkersCount = ` -SELECT count(*) as count -FROM workers; -` - - // SelectWorker represents a query to select a - // worker by hostname in the database. - SelectWorker = ` -SELECT * -FROM workers -WHERE hostname = ? -LIMIT 1; -` - - // SelectWorkerByAddress represents a query to select a - // worker by address in the database. - SelectWorkerByAddress = ` -SELECT * -FROM workers -WHERE address = ? -LIMIT 1; -` - - // DeleteWorker represents a query to - // remove a worker from the database. - DeleteWorker = ` -DELETE -FROM workers -WHERE id = ?; -` -) diff --git a/database/postgres/postgres.go b/database/postgres/postgres.go index 263b229a7..04d844675 100644 --- a/database/postgres/postgres.go +++ b/database/postgres/postgres.go @@ -13,6 +13,7 @@ import ( "github.com/go-vela/server/database/postgres/ddl" "github.com/go-vela/server/database/repo" "github.com/go-vela/server/database/user" + "github.com/go-vela/server/database/worker" "github.com/go-vela/types/constants" "github.com/sirupsen/logrus" @@ -50,6 +51,8 @@ type ( repo.RepoService // https://pkg.go.dev/github.com/go-vela/server/database/user#UserService user.UserService + // https://pkg.go.dev/github.com/go-vela/server/database/worker#WorkerService + worker.WorkerService } ) @@ -152,6 +155,8 @@ func NewTest() (*client, sqlmock.Sqlmock, error) { _mock.ExpectExec(repo.CreateOrgNameIndex).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(user.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(user.CreateUserRefreshIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(worker.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(worker.CreateHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) // create the new mock Postgres database client // @@ -268,12 +273,6 @@ func createTables(c *client) error { return fmt.Errorf("unable to create %s table: %w", constants.TableStep, err) } - // create the workers table - err = c.Postgres.Exec(ddl.CreateWorkerTable).Error - if err != nil { - return fmt.Errorf("unable to create %s table: %w", constants.TableWorker, err) - } - return nil } @@ -336,12 +335,6 @@ func createIndexes(c *client) error { return fmt.Errorf("unable to create secrets_type_org index for the %s table: %w", constants.TableSecret, err) } - // create the workers_hostname_address index for the workers table - err = c.Postgres.Exec(ddl.CreateWorkerHostnameAddressIndex).Error - if err != nil { - return fmt.Errorf("unable to create workers_hostname_address index for the %s table: %w", constants.TableWorker, err) - } - return nil } @@ -388,5 +381,17 @@ func createServices(c *client) error { return err } + // create the database agnostic worker service + // + // https://pkg.go.dev/github.com/go-vela/server/database/worker#New + c.WorkerService, err = worker.New( + worker.WithClient(c.Postgres), + worker.WithLogger(c.Logger), + worker.WithSkipCreation(c.config.SkipCreation), + ) + if err != nil { + return err + } + return nil } diff --git a/database/postgres/postgres_test.go b/database/postgres/postgres_test.go index 53375a1e7..ec80173f1 100644 --- a/database/postgres/postgres_test.go +++ b/database/postgres/postgres_test.go @@ -14,6 +14,7 @@ import ( "github.com/go-vela/server/database/postgres/ddl" "github.com/go-vela/server/database/repo" "github.com/go-vela/server/database/user" + "github.com/go-vela/server/database/worker" "github.com/go-vela/types/library" ) @@ -82,7 +83,6 @@ func TestPostgres_setupDatabase(t *testing.T) { _mock.ExpectExec(ddl.CreateSecretTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateServiceTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateStepTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateWorkerTable).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the index queries _mock.ExpectExec(ddl.CreateBuildRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -94,13 +94,19 @@ func TestPostgres_setupDatabase(t *testing.T) { _mock.ExpectExec(ddl.CreateSecretTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateSecretTypeOrgTeam).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateSecretTypeOrg).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateWorkerHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + // ensure the mock expects the pipeline queries _mock.ExpectExec(pipeline.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(pipeline.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the repo queries _mock.ExpectExec(repo.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(repo.CreateOrgNameIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the user queries _mock.ExpectExec(user.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(user.CreateUserRefreshIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the worker queries + _mock.ExpectExec(worker.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(worker.CreateHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) // setup the skip test database client _skipDatabase, _skipMock, err := NewTest() @@ -167,7 +173,6 @@ func TestPostgres_createTables(t *testing.T) { _mock.ExpectExec(ddl.CreateSecretTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateServiceTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateStepTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateWorkerTable).WillReturnResult(sqlmock.NewResult(1, 1)) tests := []struct { failure bool @@ -215,7 +220,6 @@ func TestPostgres_createIndexes(t *testing.T) { _mock.ExpectExec(ddl.CreateSecretTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateSecretTypeOrgTeam).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateSecretTypeOrg).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateWorkerHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) tests := []struct { failure bool @@ -253,13 +257,18 @@ func TestPostgres_createServices(t *testing.T) { defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - // ensure the mock expects the index queries + // ensure the mock expects the pipeline queries _mock.ExpectExec(pipeline.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(pipeline.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the repo queries _mock.ExpectExec(repo.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(repo.CreateOrgNameIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the user queries _mock.ExpectExec(user.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(user.CreateUserRefreshIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the worker queries + _mock.ExpectExec(worker.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(worker.CreateHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) tests := []struct { failure bool diff --git a/database/postgres/worker.go b/database/postgres/worker.go deleted file mode 100644 index 1d871f596..000000000 --- a/database/postgres/worker.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "errors" - - "github.com/sirupsen/logrus" - - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -// GetWorker gets a worker by hostname from the database. -func (c *client) GetWorker(hostname string) (*library.Worker, error) { - c.Logger.WithFields(logrus.Fields{ - "worker": hostname, - }).Tracef("getting worker %s from the database", hostname) - - // variable to store query results - w := new(database.Worker) - - // send query to the database and store result in variable - result := c.Postgres. - Table(constants.TableWorker). - Raw(dml.SelectWorker, hostname). - Scan(w) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - return w.ToLibrary(), result.Error -} - -// GetWorker gets a worker by address from the database. -func (c *client) GetWorkerByAddress(address string) (*library.Worker, error) { - c.Logger.Tracef("getting worker by address %s from the database", address) - - // variable to store query results - w := new(database.Worker) - - // send query to the database and store result in variable - result := c.Postgres. - Table(constants.TableWorker). - Raw(dml.SelectWorkerByAddress, address). - Scan(w) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - return w.ToLibrary(), result.Error -} - -// CreateWorker creates a new worker in the database. -func (c *client) CreateWorker(w *library.Worker) error { - c.Logger.WithFields(logrus.Fields{ - "worker": w.GetHostname(), - }).Tracef("creating worker %s in the database", w.GetHostname()) - - // cast to database type - worker := database.WorkerFromLibrary(w) - - // validate the necessary fields are populated - err := worker.Validate() - if err != nil { - return err - } - - // send query to the database - return c.Postgres. - Table(constants.TableWorker). - Create(worker).Error -} - -// UpdateWorker updates a worker in the database. -func (c *client) UpdateWorker(w *library.Worker) error { - c.Logger.WithFields(logrus.Fields{ - "worker": w.GetHostname(), - }).Tracef("updating worker %s in the database", w.GetHostname()) - - // cast to database type - worker := database.WorkerFromLibrary(w) - - // validate the necessary fields are populated - err := worker.Validate() - if err != nil { - return err - } - - // send query to the database - return c.Postgres. - Table(constants.TableWorker). - Save(worker).Error -} - -// DeleteWorker deletes a worker by unique ID from the database. -func (c *client) DeleteWorker(id int64) error { - c.Logger.Tracef("deleting worker %d in the database", id) - - // send query to the database - return c.Postgres. - Table(constants.TableWorker). - Exec(dml.DeleteWorker, id).Error -} diff --git a/database/postgres/worker_count.go b/database/postgres/worker_count.go deleted file mode 100644 index d5f867988..000000000 --- a/database/postgres/worker_count.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/constants" -) - -// GetWorkerCount gets a count of all workers from the database. -func (c *client) GetWorkerCount() (int64, error) { - c.Logger.Trace("getting count of workers from the database") - - // variable to store query results - var w int64 - - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableWorker). - Raw(dml.SelectWorkersCount). - Pluck("count", &w).Error - - return w, err -} diff --git a/database/postgres/worker_count_test.go b/database/postgres/worker_count_test.go deleted file mode 100644 index 61645f5ee..000000000 --- a/database/postgres/worker_count_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "reflect" - "testing" - - sqlmock "github.com/DATA-DOG/go-sqlmock" - - "github.com/go-vela/server/database/postgres/dml" - - "gorm.io/gorm" -) - -func TestPostgres_Client_GetWorkerCount(t *testing.T) { - // setup types - _workerOne := testWorker() - _workerOne.SetID(1) - _workerOne.SetHostname("worker_0") - _workerOne.SetAddress("localhost") - _workerOne.SetActive(true) - - _workerTwo := testWorker() - _workerTwo.SetID(2) - _workerTwo.SetHostname("worker_1") - _workerTwo.SetAddress("localhost") - _workerTwo.SetActive(true) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectWorkersCount).Statement - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) - - // ensure the mock expects the query - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetWorkerCount() - - if test.failure { - if err == nil { - t.Errorf("GetWorkerCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetWorkerCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetWorkerCount is %v, want %v", got, test.want) - } - } -} diff --git a/database/postgres/worker_list.go b/database/postgres/worker_list.go deleted file mode 100644 index 87bc66e7b..000000000 --- a/database/postgres/worker_list.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" -) - -// GetWorkerList gets a list of all workers from the database. -func (c *client) GetWorkerList() ([]*library.Worker, error) { - c.Logger.Trace("listing workers from the database") - - // variable to store query results - w := new([]database.Worker) - - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableWorker). - Raw(dml.ListWorkers). - Scan(w).Error - - // variable we want to return - workers := []*library.Worker{} - // iterate through all query results - for _, worker := range *w { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := worker - - // convert query result to library type - workers = append(workers, tmp.ToLibrary()) - } - - return workers, err -} diff --git a/database/postgres/worker_list_test.go b/database/postgres/worker_list_test.go deleted file mode 100644 index b6b3684cc..000000000 --- a/database/postgres/worker_list_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "reflect" - "testing" - - sqlmock "github.com/DATA-DOG/go-sqlmock" - - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -func TestPostgres_Client_GetWorkerList(t *testing.T) { - // setup types - _workerOne := testWorker() - _workerOne.SetID(1) - _workerOne.SetHostname("worker_0") - _workerOne.SetAddress("localhost") - _workerOne.SetActive(true) - - _workerTwo := testWorker() - _workerTwo.SetID(2) - _workerTwo.SetHostname("worker_1") - _workerTwo.SetAddress("localhost") - _workerTwo.SetActive(true) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.ListWorkers).Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "hostname", "address", "routes", "active", "last_checked_in", "build_limit"}, - ).AddRow(1, "worker_0", "localhost", "{}", true, 0, 0). - AddRow(2, "worker_1", "localhost", "{}", true, 0, 0) - - // ensure the mock expects the query - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want []*library.Worker - }{ - { - failure: false, - want: []*library.Worker{_workerOne, _workerTwo}, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetWorkerList() - - if test.failure { - if err == nil { - t.Errorf("GetWorkerList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetWorkerList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetWorkerList is %v, want %v", got, test.want) - } - } -} diff --git a/database/postgres/worker_test.go b/database/postgres/worker_test.go deleted file mode 100644 index dcc983203..000000000 --- a/database/postgres/worker_test.go +++ /dev/null @@ -1,316 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "reflect" - "testing" - - sqlmock "github.com/DATA-DOG/go-sqlmock" - "gorm.io/gorm" - - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/library" -) - -func TestPostgres_Client_GetWorker(t *testing.T) { - // setup types - _worker := testWorker() - _worker.SetID(1) - _worker.SetHostname("worker_0") - _worker.SetAddress("localhost") - _worker.SetActive(true) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectWorker, "worker_0").Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "hostname", "address", "routes", "active", "last_checked_in", "build_limit"}, - ).AddRow(1, "worker_0", "localhost", "{}", true, 0, 0) - - // ensure the mock expects the query for test case 1 - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - // ensure the mock expects the error for test case 2 - _mock.ExpectQuery(_query.SQL.String()).WillReturnError(gorm.ErrRecordNotFound) - - // setup tests - tests := []struct { - failure bool - want *library.Worker - }{ - { - failure: false, - want: _worker, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetWorker("worker_0") - - if test.failure { - if err == nil { - t.Errorf("GetWorker should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetWorker returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetWorker is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetWorkerByAddress(t *testing.T) { - // setup types - _worker := testWorker() - _worker.SetID(1) - _worker.SetHostname("worker_0") - _worker.SetAddress("localhost") - _worker.SetActive(true) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectWorkerByAddress, "localhost").Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "hostname", "address", "routes", "active", "last_checked_in", "build_limit"}, - ).AddRow(1, "worker_0", "localhost", "{}", true, 0, 0) - - // ensure the mock expects the query for test case 1 - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - // ensure the mock expects the error for test case 2 - _mock.ExpectQuery(_query.SQL.String()).WillReturnError(gorm.ErrRecordNotFound) - - // setup tests - tests := []struct { - failure bool - want *library.Worker - }{ - { - failure: false, - want: _worker, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetWorkerByAddress("localhost") - - if test.failure { - if err == nil { - t.Errorf("GetWorkerByAddress should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetWorkerByAddress returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetWorkerByAddress is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_CreateWorker(t *testing.T) { - // setup types - _worker := testWorker() - _worker.SetID(1) - _worker.SetHostname("worker_0") - _worker.SetAddress("localhost") - _worker.SetActive(true) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) - - // ensure the mock expects the query - _mock.ExpectQuery(`INSERT INTO "workers" ("hostname","address","routes","active","last_checked_in","build_limit","id") VALUES ($1,$2,$3,$4,$5,$6,$7) RETURNING "id"`). - WithArgs("worker_0", "localhost", "{}", true, nil, nil, 1). - WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := _database.CreateWorker(_worker) - - if test.failure { - if err == nil { - t.Errorf("CreateWorker should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("CreateWorker returned err: %v", err) - } - } -} - -func TestPostgres_Client_UpdateWorker(t *testing.T) { - // setup types - _worker := testWorker() - _worker.SetID(1) - _worker.SetHostname("worker_0") - _worker.SetAddress("localhost") - _worker.SetActive(true) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // ensure the mock expects the query - _mock.ExpectExec(`UPDATE "workers" SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"last_checked_in"=$5,"build_limit"=$6 WHERE "id" = $7`). - WithArgs("worker_0", "localhost", "{}", true, nil, nil, 1). - WillReturnResult(sqlmock.NewResult(1, 1)) - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := _database.UpdateWorker(_worker) - - if test.failure { - if err == nil { - t.Errorf("UpdateWorker should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("UpdateWorker returned err: %v", err) - } - } -} - -func TestPostgres_Client_DeleteWorker(t *testing.T) { - // setup types - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Exec(dml.DeleteWorker, 1).Statement - - // ensure the mock expects the query - _mock.ExpectExec(_query.SQL.String()).WillReturnResult(sqlmock.NewResult(1, 1)) - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := _database.DeleteWorker(1) - - if test.failure { - if err == nil { - t.Errorf("DeleteWorker should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("DeleteWorker returned err: %v", err) - } - } -} - -// testWorker is a test helper function to create a -// library Worker type with all fields set to their -// zero values. -func testWorker() *library.Worker { - i64 := int64(0) - str := "" - arr := []string{} - b := false - - return &library.Worker{ - ID: &i64, - Hostname: &str, - Address: &str, - Routes: &arr, - Active: &b, - LastCheckedIn: &i64, - BuildLimit: &i64, - } -} diff --git a/database/service.go b/database/service.go index 649dc0056..2191ec1c6 100644 --- a/database/service.go +++ b/database/service.go @@ -8,6 +8,7 @@ import ( "github.com/go-vela/server/database/pipeline" "github.com/go-vela/server/database/repo" "github.com/go-vela/server/database/user" + "github.com/go-vela/server/database/worker" "github.com/go-vela/types/library" ) @@ -216,27 +217,7 @@ type Service interface { // related to users stored in the database. user.UserService - // Worker Database Interface Functions - - // GetWorker defines a function that - // gets a worker by hostname. - GetWorker(string) (*library.Worker, error) - // GetWorkerByAddress defines a function that - // gets a worker by address. - GetWorkerByAddress(string) (*library.Worker, error) - // GetWorkerList defines a function that - // gets a list of all workers. - GetWorkerList() ([]*library.Worker, error) - // GetWorkerCount defines a function that - // gets the count of workers. - GetWorkerCount() (int64, error) - // CreateWorker defines a function that - // creates a new worker. - CreateWorker(*library.Worker) error - // UpdateWorker defines a function that - // updates a worker by unique ID. - UpdateWorker(*library.Worker) error - // DeleteWorker defines a function that - // deletes a worker by hostname. - DeleteWorker(int64) error + // WorkerService provides the interface for functionality + // related to workers stored in the database. + worker.WorkerService } diff --git a/database/sqlite/ddl/worker.go b/database/sqlite/ddl/worker.go deleted file mode 100644 index 0e5ba8f4a..000000000 --- a/database/sqlite/ddl/worker.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package ddl - -const ( - // CreateWorkerTable represents a query to - // create the workers table for Vela. - CreateWorkerTable = ` -CREATE TABLE -IF NOT EXISTS -workers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - hostname TEXT, - address TEXT, - routes TEXT, - active TEXT, - last_checked_in INTEGER, - build_limit INTEGER, - UNIQUE(hostname) -); -` - - // CreateWorkerHostnameAddressIndex represents a query to create an - // index on the workers table for the hostname and address columns. - CreateWorkerHostnameAddressIndex = ` -CREATE INDEX -IF NOT EXISTS -workers_hostname_address -ON workers (hostname, address); -` -) diff --git a/database/sqlite/dml/worker.go b/database/sqlite/dml/worker.go deleted file mode 100644 index 64967c9b9..000000000 --- a/database/sqlite/dml/worker.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package dml - -const ( - // ListWorkers represents a query to - // list all workers in the database. - ListWorkers = ` -SELECT * -FROM workers; -` - - // SelectWorkersCount represents a query to select the - // count of workers in the database. - SelectWorkersCount = ` -SELECT count(*) as count -FROM workers; -` - - // SelectWorker represents a query to select a - // worker in the database. - SelectWorker = ` -SELECT * -FROM workers -WHERE hostname = ? -LIMIT 1; -` - - // SelectWorkerByAddress represents a query to select a - // worker by address in the database. - SelectWorkerByAddress = ` -SELECT * -FROM workers -WHERE address = ? -LIMIT 1; -` - - // DeleteWorker represents a query to - // remove a worker from the database. - DeleteWorker = ` -DELETE -FROM workers -WHERE id = ?; -` -) diff --git a/database/sqlite/sqlite.go b/database/sqlite/sqlite.go index 5ac28bd89..1c2afcd2d 100644 --- a/database/sqlite/sqlite.go +++ b/database/sqlite/sqlite.go @@ -12,6 +12,7 @@ import ( "github.com/go-vela/server/database/repo" "github.com/go-vela/server/database/sqlite/ddl" "github.com/go-vela/server/database/user" + "github.com/go-vela/server/database/worker" "github.com/go-vela/types/constants" "github.com/sirupsen/logrus" @@ -49,6 +50,8 @@ type ( repo.RepoService // https://pkg.go.dev/github.com/go-vela/server/database/user#UserService user.UserService + // https://pkg.go.dev/github.com/go-vela/server/database/worker#WorkerService + worker.WorkerService } ) @@ -260,12 +263,6 @@ func createTables(c *client) error { return fmt.Errorf("unable to create %s table: %w", constants.TableStep, err) } - // create the workers table - err = c.Sqlite.Exec(ddl.CreateWorkerTable).Error - if err != nil { - return fmt.Errorf("unable to create %s table: %w", constants.TableWorker, err) - } - return nil } @@ -328,12 +325,6 @@ func createIndexes(c *client) error { return fmt.Errorf("unable to create secrets_type_org index for the %s table: %w", constants.TableSecret, err) } - // create the workers_hostname_address index for the workers table - err = c.Sqlite.Exec(ddl.CreateWorkerHostnameAddressIndex).Error - if err != nil { - return fmt.Errorf("unable to create workers_hostname_address index for the %s table: %w", constants.TableWorker, err) - } - return nil } @@ -380,5 +371,17 @@ func createServices(c *client) error { return err } + // create the database agnostic worker service + // + // https://pkg.go.dev/github.com/go-vela/server/database/worker#New + c.WorkerService, err = worker.New( + worker.WithClient(c.Sqlite), + worker.WithLogger(c.Logger), + worker.WithSkipCreation(c.config.SkipCreation), + ) + if err != nil { + return err + } + return nil } diff --git a/database/sqlite/worker.go b/database/sqlite/worker.go deleted file mode 100644 index 25ad1392b..000000000 --- a/database/sqlite/worker.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "errors" - - "github.com/sirupsen/logrus" - - "github.com/go-vela/server/database/sqlite/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -// GetWorker gets a worker by hostname from the database. -func (c *client) GetWorker(hostname string) (*library.Worker, error) { - c.Logger.WithFields(logrus.Fields{ - "worker": hostname, - }).Tracef("getting worker %s from the database", hostname) - - // variable to store query results - w := new(database.Worker) - - // send query to the database and store result in variable - result := c.Sqlite. - Table(constants.TableWorker). - Raw(dml.SelectWorker, hostname). - Scan(w) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - return w.ToLibrary(), result.Error -} - -// GetWorker gets a worker by address from the database. -func (c *client) GetWorkerByAddress(address string) (*library.Worker, error) { - c.Logger.Tracef("getting worker by address %s from the database", address) - - // variable to store query results - w := new(database.Worker) - - // send query to the database and store result in variable - result := c.Sqlite. - Table(constants.TableWorker). - Raw(dml.SelectWorkerByAddress, address). - Scan(w) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - return w.ToLibrary(), result.Error -} - -// CreateWorker creates a new worker in the database. -func (c *client) CreateWorker(w *library.Worker) error { - c.Logger.WithFields(logrus.Fields{ - "worker": w.GetHostname(), - }).Tracef("creating worker %s in the database", w.GetHostname()) - - // cast to database type - worker := database.WorkerFromLibrary(w) - - // validate the necessary fields are populated - err := worker.Validate() - if err != nil { - return err - } - - // send query to the database - return c.Sqlite. - Table(constants.TableWorker). - Create(worker).Error -} - -// UpdateWorker updates a worker in the database. -func (c *client) UpdateWorker(w *library.Worker) error { - c.Logger.WithFields(logrus.Fields{ - "worker": w.GetHostname(), - }).Tracef("updating worker %s in the database", w.GetHostname()) - - // cast to database type - worker := database.WorkerFromLibrary(w) - - // validate the necessary fields are populated - err := worker.Validate() - if err != nil { - return err - } - - // send query to the database - return c.Sqlite. - Table(constants.TableWorker). - Save(worker).Error -} - -// DeleteWorker deletes a worker by unique ID from the database. -func (c *client) DeleteWorker(id int64) error { - c.Logger.Tracef("deleting worker %d in the database", id) - - // send query to the database - return c.Sqlite. - Table(constants.TableWorker). - Exec(dml.DeleteWorker, id).Error -} diff --git a/database/sqlite/worker_count_test.go b/database/sqlite/worker_count_test.go deleted file mode 100644 index dbbdcfc18..000000000 --- a/database/sqlite/worker_count_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "log" - "reflect" - "testing" - - "github.com/go-vela/server/database/sqlite/ddl" - "github.com/go-vela/types/constants" -) - -func init() { - // setup the test database client - _database, err := NewTest() - if err != nil { - log.Fatalf("unable to create new sqlite test database: %v", err) - } - - // create the worker table - err = _database.Sqlite.Exec(ddl.CreateWorkerTable).Error - if err != nil { - log.Fatalf("unable to create %s table: %v", constants.TableWorker, err) - } -} - -func TestSqlite_Client_GetWorkerCount(t *testing.T) { - // setup types - _workerOne := testWorker() - _workerOne.SetID(1) - _workerOne.SetHostname("worker_0") - _workerOne.SetAddress("localhost") - _workerOne.SetActive(true) - - _workerTwo := testWorker() - _workerTwo.SetID(2) - _workerTwo.SetHostname("worker_1") - _workerTwo.SetAddress("localhost") - _workerTwo.SetActive(true) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want int64 - }{ - { - failure: false, - want: 2, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the workers table - defer _database.Sqlite.Exec("delete from workers;") - - // create the workers in the database - err := _database.CreateWorker(_workerOne) - if err != nil { - t.Errorf("unable to create test worker: %v", err) - } - - err = _database.CreateWorker(_workerTwo) - if err != nil { - t.Errorf("unable to create test worker: %v", err) - } - - got, err := _database.GetWorkerCount() - - if test.failure { - if err == nil { - t.Errorf("GetWorkerCount should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetWorkerCount returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetWorkerCount is %v, want %v", got, test.want) - } - } -} diff --git a/database/sqlite/worker_list_test.go b/database/sqlite/worker_list_test.go deleted file mode 100644 index 72916385e..000000000 --- a/database/sqlite/worker_list_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "log" - "reflect" - "testing" - - "github.com/go-vela/server/database/sqlite/ddl" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" -) - -func init() { - // setup the test database client - _database, err := NewTest() - if err != nil { - log.Fatalf("unable to create new sqlite test database: %v", err) - } - - // create the worker table - err = _database.Sqlite.Exec(ddl.CreateWorkerTable).Error - if err != nil { - log.Fatalf("unable to create %s table: %v", constants.TableWorker, err) - } -} - -func TestSqlite_Client_GetWorkerList(t *testing.T) { - // setup types - _workerOne := testWorker() - _workerOne.SetID(1) - _workerOne.SetHostname("worker_0") - _workerOne.SetAddress("localhost") - _workerOne.SetActive(true) - - _workerTwo := testWorker() - _workerTwo.SetID(2) - _workerTwo.SetHostname("worker_1") - _workerTwo.SetAddress("localhost") - _workerTwo.SetActive(true) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want []*library.Worker - }{ - { - failure: false, - want: []*library.Worker{_workerOne, _workerTwo}, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the workers table - defer _database.Sqlite.Exec("delete from workers;") - - for _, worker := range test.want { - // create the worker in the database - err := _database.CreateWorker(worker) - if err != nil { - t.Errorf("unable to create test worker: %v", err) - } - } - - got, err := _database.GetWorkerList() - - if test.failure { - if err == nil { - t.Errorf("GetWorkerList should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetWorkerList returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetWorkerList is %v, want %v", got, test.want) - } - } -} diff --git a/database/sqlite/worker_test.go b/database/sqlite/worker_test.go deleted file mode 100644 index 50c6087db..000000000 --- a/database/sqlite/worker_test.go +++ /dev/null @@ -1,311 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "reflect" - "testing" - - "github.com/go-vela/types/library" -) - -func TestSqlite_Client_GetWorker(t *testing.T) { - // setup types - _worker := testWorker() - _worker.SetID(1) - _worker.SetHostname("worker_0") - _worker.SetAddress("localhost") - _worker.SetActive(true) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want *library.Worker - }{ - { - failure: false, - want: _worker, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - if test.want != nil { - // create the worker in the database - err := _database.CreateWorker(test.want) - if err != nil { - t.Errorf("unable to create test worker: %v", err) - } - } - - got, err := _database.GetWorker("worker_0") - - // cleanup the workers table - _ = _database.Sqlite.Exec("DELETE FROM workers;") - - if test.failure { - if err == nil { - t.Errorf("GetWorker should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetWorker returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetWorker is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetWorkerByAddress(t *testing.T) { - // setup types - _worker := testWorker() - _worker.SetID(1) - _worker.SetHostname("worker_0") - _worker.SetAddress("localhost") - _worker.SetActive(true) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want *library.Worker - }{ - { - failure: false, - want: _worker, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - if test.want != nil { - // create the worker in the database - err := _database.CreateWorker(test.want) - if err != nil { - t.Errorf("unable to create test worker: %v", err) - } - } - - got, err := _database.GetWorkerByAddress("localhost") - - // cleanup the workers table - _ = _database.Sqlite.Exec("DELETE FROM workers;") - - if test.failure { - if err == nil { - t.Errorf("GetWorkerByAddress should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetWorkerByAddress returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetWorkerByAddress is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_CreateWorker(t *testing.T) { - // setup types - _worker := testWorker() - _worker.SetID(1) - _worker.SetHostname("worker_0") - _worker.SetAddress("localhost") - _worker.SetActive(true) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the workers table - defer _database.Sqlite.Exec("delete from workers;") - - err := _database.CreateWorker(_worker) - - if test.failure { - if err == nil { - t.Errorf("CreateWorker should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("CreateWorker returned err: %v", err) - } - } -} - -func TestSqlite_Client_UpdateWorker(t *testing.T) { - // setup types - _worker := testWorker() - _worker.SetID(1) - _worker.SetHostname("worker_0") - _worker.SetAddress("localhost") - _worker.SetActive(true) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the workers table - defer _database.Sqlite.Exec("delete from workers;") - - // create the worker in the database - err := _database.CreateWorker(_worker) - if err != nil { - t.Errorf("unable to create test worker: %v", err) - } - - err = _database.UpdateWorker(_worker) - - if test.failure { - if err == nil { - t.Errorf("UpdateWorker should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("UpdateWorker returned err: %v", err) - } - } -} - -func TestSqlite_Client_DeleteWorker(t *testing.T) { - // setup types - _worker := testWorker() - _worker.SetID(1) - _worker.SetHostname("worker_0") - _worker.SetAddress("localhost") - _worker.SetActive(true) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the workers table - defer _database.Sqlite.Exec("delete from workers;") - - // create the worker in the database - err := _database.CreateWorker(_worker) - if err != nil { - t.Errorf("unable to create test worker: %v", err) - } - - err = _database.DeleteWorker(1) - - if test.failure { - if err == nil { - t.Errorf("DeleteWorker should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("DeleteWorker returned err: %v", err) - } - } -} - -// testWorker is a test helper function to create a -// library Worker type with all fields set to their -// zero values. -func testWorker() *library.Worker { - i64 := int64(0) - str := "" - b := false - - var arr []string - - return &library.Worker{ - ID: &i64, - Hostname: &str, - Address: &str, - Routes: &arr, - Active: &b, - LastCheckedIn: &i64, - BuildLimit: &i64, - } -} diff --git a/database/sqlite/worker_count.go b/database/worker/count.go similarity index 53% rename from database/sqlite/worker_count.go rename to database/worker/count.go index 38b2cf7b9..8ac0f3eb5 100644 --- a/database/sqlite/worker_count.go +++ b/database/worker/count.go @@ -2,25 +2,24 @@ // // Use of this source code is governed by the LICENSE file in this repository. -package sqlite +package worker import ( - "github.com/go-vela/server/database/sqlite/dml" "github.com/go-vela/types/constants" ) -// GetWorkerCount gets a count of all workers from the database. -func (c *client) GetWorkerCount() (int64, error) { - c.Logger.Trace("getting count of workers from the database") +// CountWorkers gets the count of all workers from the database. +func (e *engine) CountWorkers() (int64, error) { + e.logger.Tracef("getting count of all workers from the database") // variable to store query results var w int64 // send query to the database and store result in variable - err := c.Sqlite. + err := e.client. Table(constants.TableWorker). - Raw(dml.SelectWorkersCount). - Pluck("count", &w).Error + Count(&w). + Error return w, err } diff --git a/database/worker/count_test.go b/database/worker/count_test.go new file mode 100644 index 000000000..bd9d4c4ac --- /dev/null +++ b/database/worker/count_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestWorker_Engine_CountWorkers(t *testing.T) { + // setup types + _workerOne := testWorker() + _workerOne.SetID(1) + _workerOne.SetHostname("worker_0") + _workerOne.SetAddress("localhost") + _workerOne.SetActive(true) + + _workerTwo := testWorker() + _workerTwo.SetID(2) + _workerTwo.SetHostname("worker_1") + _workerTwo.SetAddress("localhost") + _workerTwo.SetActive(true) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "workers"`).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateWorker(_workerOne) + if err != nil { + t.Errorf("unable to create test worker for sqlite: %v", err) + } + + err = _sqlite.CreateWorker(_workerTwo) + if err != nil { + t.Errorf("unable to create test worker for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 2, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 2, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountWorkers() + + if test.failure { + if err == nil { + t.Errorf("CountWorkers for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CountWorkers for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CountWorkers for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/worker/create.go b/database/worker/create.go new file mode 100644 index 000000000..6c62b30b6 --- /dev/null +++ b/database/worker/create.go @@ -0,0 +1,38 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// CreateWorker creates a new worker in the database. +func (e *engine) CreateWorker(w *library.Worker) error { + e.logger.WithFields(logrus.Fields{ + "worker": w.GetHostname(), + }).Tracef("creating worker %s in the database", w.GetHostname()) + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary + worker := database.WorkerFromLibrary(w) + + // validate the necessary fields are populated + // + // https://pkg.go.dev/github.com/go-vela/types/database#Worker.Validate + err := worker.Validate() + if err != nil { + return err + } + + // send query to the database + return e.client. + Table(constants.TableWorker). + Create(worker). + Error +} diff --git a/database/worker/create_test.go b/database/worker/create_test.go new file mode 100644 index 000000000..38c276d83 --- /dev/null +++ b/database/worker/create_test.go @@ -0,0 +1,73 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestWorker_Engine_CreateWorker(t *testing.T) { + // setup types + _worker := testWorker() + _worker.SetID(1) + _worker.SetHostname("worker_0") + _worker.SetAddress("localhost") + _worker.SetActive(true) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) + + // ensure the mock expects the query + _mock.ExpectQuery(`INSERT INTO "workers" +("hostname","address","routes","active","last_checked_in","build_limit","id") +VALUES ($1,$2,$3,$4,$5,$6,$7) RETURNING "id"`). + WithArgs("worker_0", "localhost", nil, true, nil, nil, 1). + WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateWorker(_worker) + + if test.failure { + if err == nil { + t.Errorf("CreateWorker for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateWorker for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/worker/delete.go b/database/worker/delete.go new file mode 100644 index 000000000..a04ebb13e --- /dev/null +++ b/database/worker/delete.go @@ -0,0 +1,30 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// DeleteWorker deletes an existing worker from the database. +func (e *engine) DeleteWorker(w *library.Worker) error { + e.logger.WithFields(logrus.Fields{ + "worker": w.GetHostname(), + }).Tracef("deleting worker %s from the database", w.GetHostname()) + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary + worker := database.WorkerFromLibrary(w) + + // send query to the database + return e.client. + Table(constants.TableWorker). + Delete(worker). + Error +} diff --git a/database/worker/delete_test.go b/database/worker/delete_test.go new file mode 100644 index 000000000..c8a9bd1be --- /dev/null +++ b/database/worker/delete_test.go @@ -0,0 +1,73 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestWorker_Engine_DeleteWorker(t *testing.T) { + // setup types + _worker := testWorker() + _worker.SetID(1) + _worker.SetHostname("worker_0") + _worker.SetAddress("localhost") + _worker.SetActive(true) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query + _mock.ExpectExec(`DELETE FROM "workers" WHERE "workers"."id" = $1`). + WithArgs(1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateWorker(_worker) + if err != nil { + t.Errorf("unable to create test worker for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err = test.database.DeleteWorker(_worker) + + if test.failure { + if err == nil { + t.Errorf("DeleteWorker for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("DeleteWorker for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/worker/get.go b/database/worker/get.go new file mode 100644 index 000000000..dd2b07ecc --- /dev/null +++ b/database/worker/get.go @@ -0,0 +1,34 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// GetWorker gets a worker by ID from the database. +func (e *engine) GetWorker(id int64) (*library.Worker, error) { + e.logger.Tracef("getting worker %d from the database", id) + + // variable to store query results + w := new(database.Worker) + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableWorker). + Where("id = ?", id). + Take(w). + Error + if err != nil { + return nil, err + } + + // return the worker + // + // https://pkg.go.dev/github.com/go-vela/types/database#Worker.ToLibrary + return w.ToLibrary(), nil +} diff --git a/database/worker/get_hostname.go b/database/worker/get_hostname.go new file mode 100644 index 000000000..6bcf42a2b --- /dev/null +++ b/database/worker/get_hostname.go @@ -0,0 +1,37 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// GetWorkerForHostname gets a worker by hostname from the database. +func (e *engine) GetWorkerForHostname(hostname string) (*library.Worker, error) { + e.logger.WithFields(logrus.Fields{ + "worker": hostname, + }).Tracef("getting worker %s from the database", hostname) + + // variable to store query results + w := new(database.Worker) + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableWorker). + Where("hostname = ?", hostname). + Take(w). + Error + if err != nil { + return nil, err + } + + // return the worker + // + // https://pkg.go.dev/github.com/go-vela/types/database#Worker.ToLibrary + return w.ToLibrary(), nil +} diff --git a/database/worker/get_hostname_test.go b/database/worker/get_hostname_test.go new file mode 100644 index 000000000..833afdc15 --- /dev/null +++ b/database/worker/get_hostname_test.go @@ -0,0 +1,85 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestWorker_Engine_GetWorkerForName(t *testing.T) { + // setup types + _worker := testWorker() + _worker.SetID(1) + _worker.SetHostname("worker_0") + _worker.SetAddress("localhost") + _worker.SetActive(true) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "hostname", "address", "routes", "active", "last_checked_in", "build_limit"}). + AddRow(1, "worker_0", "localhost", nil, true, 0, 0) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "workers" WHERE hostname = $1 LIMIT 1`).WithArgs("worker_0").WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateWorker(_worker) + if err != nil { + t.Errorf("unable to create test worker for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *library.Worker + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _worker, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _worker, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetWorkerForHostname("worker_0") + + if test.failure { + if err == nil { + t.Errorf("GetWorkerForHostname for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetWorkerForHostname for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("GetWorkerForHostname for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/worker/get_test.go b/database/worker/get_test.go new file mode 100644 index 000000000..17fd03739 --- /dev/null +++ b/database/worker/get_test.go @@ -0,0 +1,85 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestWorker_Engine_GetWorker(t *testing.T) { + // setup types + _worker := testWorker() + _worker.SetID(1) + _worker.SetHostname("worker_0") + _worker.SetAddress("localhost") + _worker.SetActive(true) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "hostname", "address", "routes", "active", "last_checked_in", "build_limit"}). + AddRow(1, "worker_0", "localhost", nil, true, 0, 0) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "workers" WHERE id = $1 LIMIT 1`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateWorker(_worker) + if err != nil { + t.Errorf("unable to create test worker for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *library.Worker + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _worker, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _worker, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetWorker(1) + + if test.failure { + if err == nil { + t.Errorf("GetWorker for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetWorker for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("GetWorker for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/worker/index.go b/database/worker/index.go new file mode 100644 index 000000000..f8f01a4b6 --- /dev/null +++ b/database/worker/index.go @@ -0,0 +1,24 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +const ( + // CreateHostnameAddressIndex represents a query to create an + // index on the workers table for the hostname and address columns. + CreateHostnameAddressIndex = ` +CREATE INDEX +IF NOT EXISTS +workers_hostname_address +ON workers (hostname, address); +` +) + +// CreateWorkerIndexes creates the indexes for the workers table in the database. +func (e *engine) CreateWorkerIndexes() error { + e.logger.Tracef("creating indexes for workers table in the database") + + // create the hostname and address columns index for the workers table + return e.client.Exec(CreateHostnameAddressIndex).Error +} diff --git a/database/worker/index_test.go b/database/worker/index_test.go new file mode 100644 index 000000000..ead204e5c --- /dev/null +++ b/database/worker/index_test.go @@ -0,0 +1,59 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestWorker_Engine_CreateWorkerIndexes(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreateHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateWorkerIndexes() + + if test.failure { + if err == nil { + t.Errorf("CreateWorkerIndexes for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateWorkerIndexes for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/sqlite/worker_list.go b/database/worker/list.go similarity index 51% rename from database/sqlite/worker_list.go rename to database/worker/list.go index e16d99071..4ec11ef3d 100644 --- a/database/sqlite/worker_list.go +++ b/database/worker/list.go @@ -2,38 +2,53 @@ // // Use of this source code is governed by the LICENSE file in this repository. -package sqlite +package worker import ( - "github.com/go-vela/server/database/sqlite/dml" "github.com/go-vela/types/constants" "github.com/go-vela/types/database" "github.com/go-vela/types/library" ) -// GetWorkerList gets a list of all workers from the database. -func (c *client) GetWorkerList() ([]*library.Worker, error) { - c.Logger.Trace("listing workers from the database") +// ListWorkers gets a list of all workers from the database. +func (e *engine) ListWorkers() ([]*library.Worker, error) { + e.logger.Trace("listing all workers from the database") - // variable to store query results + // variables to store query results and return value + count := int64(0) w := new([]database.Worker) + workers := []*library.Worker{} + + // count the results + count, err := e.CountWorkers() + if err != nil { + return nil, err + } + + // short-circuit if there are no results + if count == 0 { + return workers, nil + } // send query to the database and store result in variable - err := c.Sqlite. + err = e.client. Table(constants.TableWorker). - Raw(dml.ListWorkers). - Scan(w).Error + Find(&w). + Error + if err != nil { + return nil, err + } - // variable we want to return - workers := []*library.Worker{} // iterate through all query results for _, worker := range *w { // https://golang.org/doc/faq#closures_and_goroutines tmp := worker // convert query result to library type + // + // https://pkg.go.dev/github.com/go-vela/types/database#Worker.ToLibrary workers = append(workers, tmp.ToLibrary()) } - return workers, err + return workers, nil } diff --git a/database/worker/list_test.go b/database/worker/list_test.go new file mode 100644 index 000000000..b44c9c3d9 --- /dev/null +++ b/database/worker/list_test.go @@ -0,0 +1,103 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestWorker_Engine_ListWorkers(t *testing.T) { + // setup types + _workerOne := testWorker() + _workerOne.SetID(1) + _workerOne.SetHostname("worker_0") + _workerOne.SetAddress("localhost") + _workerOne.SetActive(true) + + _workerTwo := testWorker() + _workerTwo.SetID(2) + _workerTwo.SetHostname("worker_1") + _workerTwo.SetAddress("localhost") + _workerTwo.SetActive(true) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "workers"`).WillReturnRows(_rows) + + // create expected result in mock + _rows = sqlmock.NewRows( + []string{"id", "hostname", "address", "routes", "active", "last_checked_in", "build_limit"}). + AddRow(1, "worker_0", "localhost", nil, true, 0, 0). + AddRow(2, "worker_1", "localhost", nil, true, 0, 0) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "workers"`).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateWorker(_workerOne) + if err != nil { + t.Errorf("unable to create test worker for sqlite: %v", err) + } + + err = _sqlite.CreateWorker(_workerTwo) + if err != nil { + t.Errorf("unable to create test worker for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want []*library.Worker + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*library.Worker{_workerOne, _workerTwo}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*library.Worker{_workerOne, _workerTwo}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.ListWorkers() + + if test.failure { + if err == nil { + t.Errorf("ListWorkers for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListWorkers for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListWorkers for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/worker/opts.go b/database/worker/opts.go new file mode 100644 index 000000000..c9891ba94 --- /dev/null +++ b/database/worker/opts.go @@ -0,0 +1,44 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +// EngineOpt represents a configuration option to initialize the database engine for Workers. +type EngineOpt func(*engine) error + +// WithClient sets the gorm.io/gorm client in the database engine for Workers. +func WithClient(client *gorm.DB) EngineOpt { + return func(e *engine) error { + // set the gorm.io/gorm client in the worker engine + e.client = client + + return nil + } +} + +// WithLogger sets the github.com/sirupsen/logrus logger in the database engine for Workers. +func WithLogger(logger *logrus.Entry) EngineOpt { + return func(e *engine) error { + // set the github.com/sirupsen/logrus logger in the worker engine + e.logger = logger + + return nil + } +} + +// WithSkipCreation sets the skip creation logic in the database engine for Workers. +func WithSkipCreation(skipCreation bool) EngineOpt { + return func(e *engine) error { + // set to skip creating tables and indexes in the worker engine + e.config.SkipCreation = skipCreation + + return nil + } +} diff --git a/database/worker/opts_test.go b/database/worker/opts_test.go new file mode 100644 index 000000000..a0ebf6aa5 --- /dev/null +++ b/database/worker/opts_test.go @@ -0,0 +1,161 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "reflect" + "testing" + + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +func TestWorker_EngineOpt_WithClient(t *testing.T) { + // setup types + e := &engine{client: new(gorm.DB)} + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + want *gorm.DB + }{ + { + failure: false, + name: "client set to new database", + client: new(gorm.DB), + want: new(gorm.DB), + }, + { + failure: false, + name: "client set to nil", + client: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithClient(test.client)(e) + + if test.failure { + if err == nil { + t.Errorf("WithClient for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithClient returned err: %v", err) + } + + if !reflect.DeepEqual(e.client, test.want) { + t.Errorf("WithClient is %v, want %v", e.client, test.want) + } + }) + } +} + +func TestWorker_EngineOpt_WithLogger(t *testing.T) { + // setup types + e := &engine{logger: new(logrus.Entry)} + + // setup tests + tests := []struct { + failure bool + name string + logger *logrus.Entry + want *logrus.Entry + }{ + { + failure: false, + name: "logger set to new entry", + logger: new(logrus.Entry), + want: new(logrus.Entry), + }, + { + failure: false, + name: "logger set to nil", + logger: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithLogger(test.logger)(e) + + if test.failure { + if err == nil { + t.Errorf("WithLogger for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithLogger returned err: %v", err) + } + + if !reflect.DeepEqual(e.logger, test.want) { + t.Errorf("WithLogger is %v, want %v", e.logger, test.want) + } + }) + } +} + +func TestWorker_EngineOpt_WithSkipCreation(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + skipCreation bool + want bool + }{ + { + failure: false, + name: "skip creation set to true", + skipCreation: true, + want: true, + }, + { + failure: false, + name: "skip creation set to false", + skipCreation: false, + want: false, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithSkipCreation(test.skipCreation)(e) + + if test.failure { + if err == nil { + t.Errorf("WithSkipCreation for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithSkipCreation returned err: %v", err) + } + + if !reflect.DeepEqual(e.config.SkipCreation, test.want) { + t.Errorf("WithSkipCreation is %v, want %v", e.config.SkipCreation, test.want) + } + }) + } +} diff --git a/database/worker/service.go b/database/worker/service.go new file mode 100644 index 000000000..b02c6f854 --- /dev/null +++ b/database/worker/service.go @@ -0,0 +1,43 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "github.com/go-vela/types/library" +) + +// WorkerService represents the Vela interface for worker +// functions with the supported Database backends. +// +//nolint:revive // ignore name stutter +type WorkerService interface { + // Worker Data Definition Language Functions + // + // https://en.wikipedia.org/wiki/Data_definition_language + + // CreateWorkerIndexes defines a function that creates the indexes for the workers table. + CreateWorkerIndexes() error + // CreateWorkerTable defines a function that creates the workers table. + CreateWorkerTable(string) error + + // Worker Data Manipulation Language Functions + // + // https://en.wikipedia.org/wiki/Data_manipulation_language + + // CountWorkers defines a function that gets the count of all workers. + CountWorkers() (int64, error) + // CreateWorker defines a function that creates a new worker. + CreateWorker(*library.Worker) error + // DeleteWorker defines a function that deletes an existing worker. + DeleteWorker(*library.Worker) error + // GetWorker defines a function that gets a worker by ID. + GetWorker(int64) (*library.Worker, error) + // GetWorkerForHostname defines a function that gets a worker by hostname. + GetWorkerForHostname(string) (*library.Worker, error) + // ListWorkers defines a function that gets a list of all workers. + ListWorkers() ([]*library.Worker, error) + // UpdateWorker defines a function that updates an existing worker. + UpdateWorker(*library.Worker) error +} diff --git a/database/worker/table.go b/database/worker/table.go new file mode 100644 index 000000000..5cecf109d --- /dev/null +++ b/database/worker/table.go @@ -0,0 +1,60 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "github.com/go-vela/types/constants" +) + +const ( + // CreatePostgresTable represents a query to create the Postgres workers table. + CreatePostgresTable = ` +CREATE TABLE +IF NOT EXISTS +workers ( + id SERIAL PRIMARY KEY, + hostname VARCHAR(250), + address VARCHAR(250), + routes VARCHAR(1000), + active BOOLEAN, + last_checked_in INTEGER, + build_limit INTEGER, + UNIQUE(hostname) +); +` + + // CreateSqliteTable represents a query to create the Sqlite workers table. + CreateSqliteTable = ` +CREATE TABLE +IF NOT EXISTS +workers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hostname TEXT, + address TEXT, + routes TEXT, + active BOOLEAN, + last_checked_in INTEGER, + build_limit INTEGER, + UNIQUE(hostname) +); +` +) + +// CreateWorkerTable creates the workers table in the database. +func (e *engine) CreateWorkerTable(driver string) error { + e.logger.Tracef("creating workers table in the database") + + // handle the driver provided to create the table + switch driver { + case constants.DriverPostgres: + // create the workers table for Postgres + return e.client.Exec(CreatePostgresTable).Error + case constants.DriverSqlite: + fallthrough + default: + // create the workers table for Sqlite + return e.client.Exec(CreateSqliteTable).Error + } +} diff --git a/database/worker/table_test.go b/database/worker/table_test.go new file mode 100644 index 000000000..681a267f2 --- /dev/null +++ b/database/worker/table_test.go @@ -0,0 +1,59 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestWorker_Engine_CreateWorkerTable(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateWorkerTable(test.name) + + if test.failure { + if err == nil { + t.Errorf("CreateWorkerTable for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateWorkerTable for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/worker/update.go b/database/worker/update.go new file mode 100644 index 000000000..b0e475273 --- /dev/null +++ b/database/worker/update.go @@ -0,0 +1,38 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" +) + +// UpdateWorker updates an existing worker in the database. +func (e *engine) UpdateWorker(w *library.Worker) error { + e.logger.WithFields(logrus.Fields{ + "worker": w.GetHostname(), + }).Tracef("updating worker %s in the database", w.GetHostname()) + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary + worker := database.WorkerFromLibrary(w) + + // validate the necessary fields are populated + // + // https://pkg.go.dev/github.com/go-vela/types/database#Worker.Validate + err := worker.Validate() + if err != nil { + return err + } + + // send query to the database + return e.client. + Table(constants.TableWorker). + Save(worker). + Error +} diff --git a/database/worker/update_test.go b/database/worker/update_test.go new file mode 100644 index 000000000..88644678a --- /dev/null +++ b/database/worker/update_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestWorker_Engine_UpdateWorker(t *testing.T) { + // setup types + _worker := testWorker() + _worker.SetID(1) + _worker.SetHostname("worker_0") + _worker.SetAddress("localhost") + _worker.SetActive(true) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query + _mock.ExpectExec(`UPDATE "workers" +SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"last_checked_in"=$5,"build_limit"=$6 +WHERE "id" = $7`). + WithArgs("worker_0", "localhost", nil, true, nil, nil, 1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateWorker(_worker) + if err != nil { + t.Errorf("unable to create test worker for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err = test.database.UpdateWorker(_worker) + + if test.failure { + if err == nil { + t.Errorf("UpdateWorker for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("UpdateWorker for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/worker/worker.go b/database/worker/worker.go new file mode 100644 index 000000000..d870fb1ba --- /dev/null +++ b/database/worker/worker.go @@ -0,0 +1,80 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "fmt" + + "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +type ( + // config represents the settings required to create the engine that implements the WorkerService interface. + config struct { + // specifies to skip creating tables and indexes for the Worker engine + SkipCreation bool + } + + // engine represents the worker functionality that implements the WorkerService interface. + engine struct { + // engine configuration settings used in worker functions + config *config + + // gorm.io/gorm database client used in worker functions + // + // https://pkg.go.dev/gorm.io/gorm#DB + client *gorm.DB + + // sirupsen/logrus logger used in worker functions + // + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry + logger *logrus.Entry + } +) + +// New creates and returns a Vela service for integrating with workers in the database. +// +//nolint:revive // ignore returning unexported engine +func New(opts ...EngineOpt) (*engine, error) { + // create new Worker engine + e := new(engine) + + // create new fields + e.client = new(gorm.DB) + e.config = new(config) + e.logger = new(logrus.Entry) + + // apply all provided configuration options + for _, opt := range opts { + err := opt(e) + if err != nil { + return nil, err + } + } + + // check if we should skip creating worker database objects + if e.config.SkipCreation { + e.logger.Warning("skipping creation of workers table and indexes in the database") + + return e, nil + } + + // create the workers table + err := e.CreateWorkerTable(e.client.Config.Dialector.Name()) + if err != nil { + return nil, fmt.Errorf("unable to create %s table: %w", constants.TableWorker, err) + } + + // create the indexes for the workers table + err = e.CreateWorkerIndexes() + if err != nil { + return nil, fmt.Errorf("unable to create indexes for %s table: %w", constants.TableWorker, err) + } + + return e, nil +} diff --git a/database/worker/worker_test.go b/database/worker/worker_test.go new file mode 100644 index 000000000..48deabe54 --- /dev/null +++ b/database/worker/worker_test.go @@ -0,0 +1,181 @@ +// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package worker + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" + + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestWorker_New(t *testing.T) { + // setup types + logger := logrus.NewEntry(logrus.StandardLogger()) + + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + defer _sql.Close() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + _config := &gorm.Config{SkipDefaultTransaction: true} + + _postgres, err := gorm.Open(postgres.New(postgres.Config{Conn: _sql}), _config) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _sqlite, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), _config) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + defer func() { _sql, _ := _sqlite.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + key string + logger *logrus.Entry + skipCreation bool + want *engine + }{ + { + failure: false, + name: "postgres", + client: _postgres, + logger: logger, + skipCreation: false, + want: &engine{ + client: _postgres, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + { + failure: false, + name: "sqlite3", + client: _sqlite, + logger: logger, + skipCreation: false, + want: &engine{ + client: _sqlite, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := New( + WithClient(test.client), + WithLogger(test.logger), + WithSkipCreation(test.skipCreation), + ) + + if test.failure { + if err == nil { + t.Errorf("New for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("New for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("New for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} + +// testPostgres is a helper function to create a Postgres engine for testing. +func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { + // create the new mock sql database + // + // https://pkg.go.dev/github.com/DATA-DOG/go-sqlmock#New + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + // create the new mock Postgres database client + // + // https://pkg.go.dev/gorm.io/gorm#Open + _postgres, err := gorm.Open( + postgres.New(postgres.Config{Conn: _sql}), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _engine, err := New( + WithClient(_postgres), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new postgres worker engine: %v", err) + } + + return _engine, _mock +} + +// testSqlite is a helper function to create a Sqlite engine for testing. +func testSqlite(t *testing.T) *engine { + _sqlite, err := gorm.Open( + sqlite.Open("file::memory:?cache=shared"), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + _engine, err := New( + WithClient(_sqlite), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new sqlite worker engine: %v", err) + } + + return _engine +} + +// testWorker is a test helper function to create a library +// Worker type with all fields set to their zero values. +func testWorker() *library.Worker { + return &library.Worker{ + ID: new(int64), + Hostname: new(string), + Address: new(string), + Routes: new([]string), + Active: new(bool), + BuildLimit: new(int64), + LastCheckedIn: new(int64), + } +} diff --git a/router/middleware/executors/executors.go b/router/middleware/executors/executors.go index acd115537..96155eacb 100644 --- a/router/middleware/executors/executors.go +++ b/router/middleware/executors/executors.go @@ -30,7 +30,7 @@ func Establish() gin.HandlerFunc { e := new([]library.Executor) b := build.Retrieve(c) // retrieve the worker - w, err := database.FromContext(c).GetWorker(b.GetHost()) + w, err := database.FromContext(c).GetWorkerForHostname(b.GetHost()) if err != nil { retErr := fmt.Errorf("unable to get worker: %w", err) util.HandleError(c, http.StatusNotFound, retErr) diff --git a/router/middleware/worker/worker.go b/router/middleware/worker/worker.go index 2f68b811f..5afc8b3bf 100644 --- a/router/middleware/worker/worker.go +++ b/router/middleware/worker/worker.go @@ -33,7 +33,7 @@ func Establish() gin.HandlerFunc { logrus.Debugf("Reading worker %s", wParam) - w, err := database.FromContext(c).GetWorker(wParam) + w, err := database.FromContext(c).GetWorkerForHostname(wParam) if err != nil { retErr := fmt.Errorf("unable to read worker %s: %w", wParam, err) util.HandleError(c, http.StatusNotFound, retErr)