From 1b8ab7c704d04893de4feef88c6b7ee036032eb3 Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Wed, 31 May 2023 21:14:40 -0500 Subject: [PATCH 01/13] feat(database): add agnostic engine --- database/database.go | 58 +++++++++++++++++++++++++++- database/database_test.go | 79 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 134 insertions(+), 3 deletions(-) diff --git a/database/database.go b/database/database.go index ad26dd82a..d61cc5fc1 100644 --- a/database/database.go +++ b/database/database.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. @@ -6,10 +6,64 @@ package database import ( "fmt" + "time" + "github.com/go-vela/server/database/build" + "github.com/go-vela/server/database/hook" + "github.com/go-vela/server/database/log" + "github.com/go-vela/server/database/pipeline" + "github.com/go-vela/server/database/repo" + "github.com/go-vela/server/database/schedule" + "github.com/go-vela/server/database/secret" + "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/step" + "github.com/go-vela/server/database/user" + "github.com/go-vela/server/database/worker" "github.com/go-vela/types/constants" - "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +type ( + // Config represents the settings required to create the engine that implements the Interface. + Config struct { + // specifies the address to use for the database client + Address string + // specifies the level of compression to use for the database client + CompressionLevel int + // specifies the connection duration to use for the database client + ConnectionLife time.Duration + // specifies the maximum idle connections for the database client + ConnectionIdle int + // specifies the maximum open connections for the database client + ConnectionOpen int + // specifies the driver to use for the database client + Driver string + // specifies the encryption key to use for the database client + EncryptionKey string + // specifies to skip creating tables and indexes for the database client + SkipCreation bool + } + + // engine represents the functionality that implements the Interface. + engine struct { + Config *Config + Database *gorm.DB + Logger *logrus.Entry + + build.BuildInterface + hook.HookInterface + log.LogInterface + pipeline.PipelineInterface + repo.RepoInterface + schedule.ScheduleInterface + secret.SecretInterface + service.ServiceInterface + step.StepInterface + user.UserInterface + worker.WorkerInterface + } ) // New creates and returns a Vela service capable of diff --git a/database/database_test.go b/database/database_test.go index 14081a382..77de76d5b 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. @@ -7,6 +7,13 @@ package database import ( "testing" "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/sirupsen/logrus" + + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" ) func TestDatabase_New(t *testing.T) { @@ -86,3 +93,73 @@ func TestDatabase_New(t *testing.T) { } } } + +// testPostgres is a helper function to create a Postgres engine for testing. +func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { + // create the engine with test configuration + _engine := &engine{ + Config: &Config{ + CompressionLevel: 3, + ConnectionLife: 30 * time.Minute, + ConnectionIdle: 2, + ConnectionOpen: 0, + Driver: "postgres", + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + Logger: logrus.NewEntry(logrus.StandardLogger()), + } + + // create the new mock sql database + _sql, _mock, err := sqlmock.New( + sqlmock.MonitorPingsOption(true), + sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual), + ) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + // ensure the mock expects the ping + _mock.ExpectPing() + + // create the new mock Postgres database client + _engine.Database, err = gorm.Open( + postgres.New(postgres.Config{Conn: _sql}), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new test postgres database: %v", err) + } + + return _engine, _mock +} + +// testSqlite is a helper function to create a Sqlite engine for testing. +func testSqlite(t *testing.T) *engine { + var err error + + // create the engine with test configuration + _engine := &engine{ + Config: &Config{ + Address: "file::memory:?cache=shared", + CompressionLevel: 3, + ConnectionLife: 30 * time.Minute, + ConnectionIdle: 2, + ConnectionOpen: 0, + Driver: "sqlite3", + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + Logger: logrus.NewEntry(logrus.StandardLogger()), + } + + // create the new mock Sqlite database client + _engine.Database, err = gorm.Open( + sqlite.Open(_engine.Config.Address), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new test sqlite database: %v", err) + } + + return _engine +} From 19bf93bdc0ca1035bf4968b5d8c27bc4ba9325b0 Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Wed, 31 May 2023 21:15:52 -0500 Subject: [PATCH 02/13] feat(database): add config.Validate() --- database/validate.go | 75 +++++++++++++++++ database/validate_test.go | 164 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+) create mode 100644 database/validate.go create mode 100644 database/validate_test.go diff --git a/database/validate.go b/database/validate.go new file mode 100644 index 000000000..c16c07cd2 --- /dev/null +++ b/database/validate.go @@ -0,0 +1,75 @@ +// Copyright (c) 2023 Target Brands, Ine. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "fmt" + "strings" + + "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" +) + +// Validate verifies the required fields from the provided configuration are populated correctly. +func (c *Config) Validate() error { + logrus.Trace("validating database configuration for engine") + + // verify a database driver was provided + if len(c.Driver) == 0 { + return fmt.Errorf("no database driver provided") + } + + // verify a database address was provided + if len(c.Address) == 0 { + return fmt.Errorf("no database address provided") + } + + // check if the database address has a trailing slash + if strings.HasSuffix(c.Address, "/") { + return fmt.Errorf("invalid database address provided: address must not have trailing slash") + } + + // verify a database encryption key was provided + if len(c.EncryptionKey) == 0 { + return fmt.Errorf("no database encryption key provided") + } + + // check the database encryption key length - enforce AES-256 by forcing 32 characters in the key + if len(c.EncryptionKey) != 32 { + return fmt.Errorf("invalid database encryption key provided: key length (%d) must be 32 characters", len(c.EncryptionKey)) + } + + // verify the database compression level is valid + switch c.CompressionLevel { + case constants.CompressionNegOne: + fallthrough + case constants.CompressionZero: + fallthrough + case constants.CompressionOne: + fallthrough + case constants.CompressionTwo: + fallthrough + case constants.CompressionThree: + fallthrough + case constants.CompressionFour: + fallthrough + case constants.CompressionFive: + fallthrough + case constants.CompressionSix: + fallthrough + case constants.CompressionSeven: + fallthrough + case constants.CompressionEight: + fallthrough + case constants.CompressionNine: + break + default: + return fmt.Errorf("invalid database compression level provided: level (%d) must be between %d and %d", + c.CompressionLevel, constants.CompressionNegOne, constants.CompressionNine, + ) + } + + return nil +} diff --git a/database/validate_test.go b/database/validate_test.go new file mode 100644 index 000000000..11be746a8 --- /dev/null +++ b/database/validate_test.go @@ -0,0 +1,164 @@ +// Copyright (c) 2023 Target Brands, Ine. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "testing" + "time" +) + +func TestDatabase_Config_Validate(t *testing.T) { + tests := []struct { + failure bool + name string + config *Config + }{ + { + name: "success with postgres", + failure: false, + config: &Config{ + Driver: "postgres", + Address: "postgres://foo:bar@localhost:5432/vela", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "success with sqlite3", + failure: false, + config: &Config{ + Driver: "sqlite3", + Address: "file::memory:?cache=shared", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "success with negative compression level", + failure: false, + config: &Config{ + Driver: "postgres", + Address: "postgres://foo:bar@localhost:5432/vela", + CompressionLevel: -1, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "failure with empty driver", + failure: true, + config: &Config{ + Driver: "", + Address: "postgres://foo:bar@localhost:5432/vela", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "failure with empty address", + failure: true, + config: &Config{ + Driver: "postgres", + Address: "", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "failure with invalid address", + failure: true, + config: &Config{ + Driver: "postgres", + Address: "postgres://foo:bar@localhost:5432/vela/", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "failure with invalid compression level", + failure: true, + config: &Config{ + Driver: "postgres", + Address: "postgres://foo:bar@localhost:5432/vela", + CompressionLevel: 10, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }, + }, + { + name: "failure with empty encryption key", + failure: true, + config: &Config{ + Driver: "postgres", + Address: "postgres://foo:bar@localhost:5432/vela", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "", + SkipCreation: false, + }, + }, + { + name: "failure with invalid encryption key", + failure: true, + config: &Config{ + Driver: "postgres", + Address: "postgres://foo:bar@localhost:5432/vela", + CompressionLevel: 3, + ConnectionLife: 10 * time.Second, + ConnectionIdle: 5, + ConnectionOpen: 20, + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0", + SkipCreation: false, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.config.Validate() + + if test.failure { + if err == nil { + t.Errorf("Validate for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("Validate for %s returned err: %v", test.name, err) + } + }) + } +} From f820f6a797c9e9e6ac946a130ba26cb6dd151848 Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Wed, 31 May 2023 21:26:30 -0500 Subject: [PATCH 03/13] feat(database): add engine.Close() --- database/close.go | 18 ++++++++++ database/close_test.go | 82 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 database/close.go create mode 100644 database/close_test.go diff --git a/database/close.go b/database/close.go new file mode 100644 index 000000000..0596b972e --- /dev/null +++ b/database/close.go @@ -0,0 +1,18 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +// Close stops and terminates the connection to the database. +func (e *engine) Close() error { + e.Logger.Tracef("closing connection to the %s database", e.Driver()) + + // capture database/sql database from gorm.io/gorm database + _sql, err := e.Database.DB() + if err != nil { + return err + } + + return _sql.Close() +} diff --git a/database/close_test.go b/database/close_test.go new file mode 100644 index 000000000..bb9c6635b --- /dev/null +++ b/database/close_test.go @@ -0,0 +1,82 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "testing" + + "github.com/sirupsen/logrus" + "gorm.io/gorm" +) + +func TestDatabase_Engine_Close(t *testing.T) { + _postgres, _mock := testPostgres(t) + defer _postgres.Close() + // ensure the mock expects the close + _mock.ExpectClose() + + // create a test database without mocking the call + _unmocked, _ := testPostgres(t) + + _sqlite := testSqlite(t) + defer _sqlite.Close() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + name: "success with postgres", + failure: false, + database: _postgres, + }, + { + name: "success with sqlite3", + failure: false, + database: _sqlite, + }, + { + name: "failure without mocked call", + failure: true, + database: _unmocked, + }, + { + name: "failure with invalid gorm database", + failure: true, + database: &engine{ + Config: &Config{ + Driver: "invalid", + }, + Database: &gorm.DB{ + Config: &gorm.Config{ + ConnPool: nil, + }, + }, + Logger: logrus.NewEntry(logrus.StandardLogger()), + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.Close() + + if test.failure { + if err == nil { + t.Errorf("Close for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("Close for %s returned err: %v", test.name, err) + } + }) + } +} From 5d5490cce75e592e947df6b808713278c23b4576 Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Wed, 31 May 2023 21:26:59 -0500 Subject: [PATCH 04/13] feat(database): add engine.Driver() --- database/driver.go | 10 +++++++++ database/driver_test.go | 47 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 database/driver.go create mode 100644 database/driver_test.go diff --git a/database/driver.go b/database/driver.go new file mode 100644 index 000000000..a81662846 --- /dev/null +++ b/database/driver.go @@ -0,0 +1,10 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +// Driver outputs the configured database driver. +func (e *engine) Driver() string { + return e.Config.Driver +} diff --git a/database/driver_test.go b/database/driver_test.go new file mode 100644 index 000000000..6d382d8cd --- /dev/null +++ b/database/driver_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "strings" + "testing" +) + +func TestDatabase_Engine_Driver(t *testing.T) { + _postgres, _ := testPostgres(t) + defer _postgres.Close() + + _sqlite := testSqlite(t) + defer _sqlite.Close() + + // setup tests + tests := []struct { + name string + database *engine + want string + }{ + { + name: "success with postgres", + database: _postgres, + want: "postgres", + }, + { + name: "success with sqlite3", + database: _sqlite, + want: "sqlite3", + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := test.database.Driver() + + if !strings.EqualFold(got, test.want) { + t.Errorf("Driver for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} From ab04f2fba92464bd604be1c7f60ecf64a47412ba Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Wed, 31 May 2023 21:37:17 -0500 Subject: [PATCH 05/13] chore: minor fixes --- database/close_test.go | 1 + database/database.go | 18 +++++++++--------- database/validate_test.go | 1 + 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/database/close_test.go b/database/close_test.go index bb9c6635b..b555e78d5 100644 --- a/database/close_test.go +++ b/database/close_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/sirupsen/logrus" + "gorm.io/gorm" ) diff --git a/database/database.go b/database/database.go index d61cc5fc1..bf0ca2724 100644 --- a/database/database.go +++ b/database/database.go @@ -28,21 +28,21 @@ import ( type ( // Config represents the settings required to create the engine that implements the Interface. Config struct { - // specifies the address to use for the database client + // specifies the address to use for the database engine Address string - // specifies the level of compression to use for the database client + // specifies the level of compression to use for the database engine CompressionLevel int - // specifies the connection duration to use for the database client - ConnectionLife time.Duration - // specifies the maximum idle connections for the database client + // specifies the maximum idle connections for the database engine ConnectionIdle int - // specifies the maximum open connections for the database client + // specifies the connection duration to use for the database engine + ConnectionLife time.Duration + // specifies the maximum open connections for the database engine ConnectionOpen int - // specifies the driver to use for the database client + // specifies the driver to use for the database engine Driver string - // specifies the encryption key to use for the database client + // specifies the encryption key to use for the database engine EncryptionKey string - // specifies to skip creating tables and indexes for the database client + // specifies to skip creating tables and indexes for the database engine SkipCreation bool } diff --git a/database/validate_test.go b/database/validate_test.go index 11be746a8..356705fe2 100644 --- a/database/validate_test.go +++ b/database/validate_test.go @@ -10,6 +10,7 @@ import ( ) func TestDatabase_Config_Validate(t *testing.T) { + // setup tests tests := []struct { failure bool name string From a51f84c7afff39fabde7ae094a792c2f6c8acbc3 Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Thu, 1 Jun 2023 08:14:15 -0500 Subject: [PATCH 06/13] feat(database): add engine.Ping() --- database/ping.go | 42 ++++++++++++++++++++++ database/ping_test.go | 82 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 database/ping.go create mode 100644 database/ping_test.go diff --git a/database/ping.go b/database/ping.go new file mode 100644 index 000000000..7ac1a09c2 --- /dev/null +++ b/database/ping.go @@ -0,0 +1,42 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "fmt" + "time" +) + +// Ping sends a "ping" request with backoff to the database. +func (e *engine) Ping() error { + e.Logger.Tracef("sending ping request to the %s database", e.Driver()) + + // create a loop to attempt ping requests 5 times + for i := 0; i < 5; i++ { + // capture database/sql database from gorm.io/gorm database + _sql, err := e.Database.DB() + if err != nil { + return err + } + + // send ping request to database + err = _sql.Ping() + if err != nil { + // create the duration of time to sleep for before attempting to retry + duration := time.Duration(i+1) * time.Second + + e.Logger.Warnf("unable to ping %s database - retrying in %v", e.Driver(), duration) + + // sleep for loop iteration in seconds + time.Sleep(duration) + + continue + } + + return nil + } + + return fmt.Errorf("unable to successfully ping %s database", e.Driver()) +} diff --git a/database/ping_test.go b/database/ping_test.go new file mode 100644 index 000000000..7a34ad122 --- /dev/null +++ b/database/ping_test.go @@ -0,0 +1,82 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "testing" + + "github.com/sirupsen/logrus" + "gorm.io/gorm" +) + +func TestDatabase_Engine_Ping(t *testing.T) { + _postgres, _mock := testPostgres(t) + defer _postgres.Close() + // ensure the mock expects the ping + _mock.ExpectPing() + + // create a test database without mocking the call + _unmocked, _ := testPostgres(t) + + _sqlite := testSqlite(t) + defer _sqlite.Close() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + name: "success with postgres", + failure: false, + database: _postgres, + }, + { + name: "success with sqlite", + failure: false, + database: _sqlite, + }, + { + name: "failure without mocked call", + failure: true, + database: _unmocked, + }, + { + name: "failure with invalid gorm database", + failure: true, + database: &engine{ + Config: &Config{ + Driver: "invalid", + }, + Database: &gorm.DB{ + Config: &gorm.Config{ + ConnPool: nil, + }, + }, + Logger: logrus.NewEntry(logrus.StandardLogger()), + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.Ping() + + if test.failure { + if err == nil { + t.Errorf("Ping for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("Ping for %s returned err: %v", test.name, err) + } + }) + } +} From b8858a22bf762b6eb37a15bd5a404c90e1bc50a9 Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Thu, 1 Jun 2023 08:17:36 -0500 Subject: [PATCH 07/13] feat(database): add engine.NewResources() --- database/resource.go | 143 ++++++++++++++++++++++++++++++++++++++ database/resource_test.go | 112 +++++++++++++++++++++++++++++ 2 files changed, 255 insertions(+) create mode 100644 database/resource.go create mode 100644 database/resource_test.go diff --git a/database/resource.go b/database/resource.go new file mode 100644 index 000000000..93122eadc --- /dev/null +++ b/database/resource.go @@ -0,0 +1,143 @@ +// Copyright (c) 2023 Target Brands, Ine. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "github.com/go-vela/server/database/build" + "github.com/go-vela/server/database/hook" + "github.com/go-vela/server/database/log" + "github.com/go-vela/server/database/pipeline" + "github.com/go-vela/server/database/repo" + "github.com/go-vela/server/database/schedule" + "github.com/go-vela/server/database/secret" + "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/step" + "github.com/go-vela/server/database/user" + "github.com/go-vela/server/database/worker" +) + +// NewResources creates and returns the database agnostic engines for resources. +func (e *engine) NewResources() error { + var err error + + // create the database agnostic engine for builds + e.BuildInterface, err = build.New( + build.WithClient(e.Database), + build.WithLogger(e.Logger), + build.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for hooks + e.HookInterface, err = hook.New( + hook.WithClient(e.Database), + hook.WithLogger(e.Logger), + hook.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for logs + e.LogInterface, err = log.New( + log.WithClient(e.Database), + log.WithCompressionLevel(e.Config.CompressionLevel), + log.WithLogger(e.Logger), + log.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for pipelines + e.PipelineInterface, err = pipeline.New( + pipeline.WithClient(e.Database), + pipeline.WithCompressionLevel(e.Config.CompressionLevel), + pipeline.WithLogger(e.Logger), + pipeline.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for repos + e.RepoInterface, err = repo.New( + repo.WithClient(e.Database), + repo.WithEncryptionKey(e.Config.EncryptionKey), + repo.WithLogger(e.Logger), + repo.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for schedules + e.ScheduleInterface, err = schedule.New( + schedule.WithClient(e.Database), + schedule.WithLogger(e.Logger), + schedule.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for secrets + // + // https://pkg.go.dev/github.com/go-vela/server/database/secret#New + e.SecretInterface, err = secret.New( + secret.WithClient(e.Database), + secret.WithEncryptionKey(e.Config.EncryptionKey), + secret.WithLogger(e.Logger), + secret.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for services + e.ServiceInterface, err = service.New( + service.WithClient(e.Database), + service.WithLogger(e.Logger), + service.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for steps + e.StepInterface, err = step.New( + step.WithClient(e.Database), + step.WithLogger(e.Logger), + step.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for users + e.UserInterface, err = user.New( + user.WithClient(e.Database), + user.WithEncryptionKey(e.Config.EncryptionKey), + user.WithLogger(e.Logger), + user.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + // create the database agnostic engine for workers + e.WorkerInterface, err = worker.New( + worker.WithClient(e.Database), + worker.WithLogger(e.Logger), + worker.WithSkipCreation(e.Config.SkipCreation), + ) + if err != nil { + return err + } + + return nil +} diff --git a/database/resource_test.go b/database/resource_test.go new file mode 100644 index 000000000..36a4694e6 --- /dev/null +++ b/database/resource_test.go @@ -0,0 +1,112 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package database + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/server/database/build" + "github.com/go-vela/server/database/hook" + "github.com/go-vela/server/database/log" + "github.com/go-vela/server/database/pipeline" + "github.com/go-vela/server/database/repo" + "github.com/go-vela/server/database/schedule" + "github.com/go-vela/server/database/secret" + "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/step" + "github.com/go-vela/server/database/user" + "github.com/go-vela/server/database/worker" +) + +func TestDatabase_Engine_NewResources(t *testing.T) { + _postgres, _mock := testPostgres(t) + defer _postgres.Close() + + // ensure the mock expects the build queries + _mock.ExpectExec(build.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(build.CreateStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the hook queries + _mock.ExpectExec(hook.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(hook.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the log queries + _mock.ExpectExec(log.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(log.CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the pipeline queries + _mock.ExpectExec(pipeline.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(pipeline.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the repo queries + _mock.ExpectExec(repo.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(repo.CreateOrgNameIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the schedule queries + _mock.ExpectExec(schedule.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(schedule.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the secret queries + _mock.ExpectExec(secret.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(secret.CreateTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(secret.CreateTypeOrgTeam).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(secret.CreateTypeOrg).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the service queries + _mock.ExpectExec(service.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the step queries + _mock.ExpectExec(step.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the user queries + _mock.ExpectExec(user.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(user.CreateUserRefreshIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the worker queries + _mock.ExpectExec(worker.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(worker.CreateHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + // create a test database without mocking the call + _unmocked, _ := testPostgres(t) + + _sqlite := testSqlite(t) + defer _sqlite.Close() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + name: "success with postgres", + failure: false, + database: _postgres, + }, + { + name: "success with sqlite3", + failure: false, + database: _sqlite, + }, + { + name: "failure without mocked call", + failure: true, + database: _unmocked, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.NewResources() + + if test.failure { + if err == nil { + t.Errorf("NewResources for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("NewResources for %s returned err: %v", test.name, err) + } + }) + } +} From 6f52e1f3983f3a6c5a90d2943bf4e63ba48f2762 Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Tue, 6 Jun 2023 08:02:14 -0500 Subject: [PATCH 08/13] refactor(database): use agnostic engine --- database/database.go | 112 +++++++++++++++++++++++++++----------- database/database_test.go | 45 ++++++++------- 2 files changed, 107 insertions(+), 50 deletions(-) diff --git a/database/database.go b/database/database.go index bf0ca2724..d60cb5189 100644 --- a/database/database.go +++ b/database/database.go @@ -22,27 +22,29 @@ import ( "github.com/go-vela/types/constants" "github.com/sirupsen/logrus" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" "gorm.io/gorm" ) type ( // Config represents the settings required to create the engine that implements the Interface. Config struct { - // specifies the address to use for the database engine + // specifies the address to use for the database client Address string - // specifies the level of compression to use for the database engine + // specifies the level of compression to use for the database client CompressionLevel int - // specifies the maximum idle connections for the database engine - ConnectionIdle int - // specifies the connection duration to use for the database engine + // specifies the connection duration to use for the database client ConnectionLife time.Duration - // specifies the maximum open connections for the database engine + // specifies the maximum idle connections for the database client + ConnectionIdle int + // specifies the maximum open connections for the database client ConnectionOpen int - // specifies the driver to use for the database engine + // specifies the driver to use for the database client Driver string - // specifies the encryption key to use for the database engine + // specifies the encryption key to use for the database client EncryptionKey string - // specifies to skip creating tables and indexes for the database engine + // specifies to skip creating tables and indexes for the database client SkipCreation bool } @@ -66,38 +68,86 @@ type ( } ) -// New creates and returns a Vela service capable of -// integrating with the configured database provider. +// New creates and returns an engine capable of integrating with the configured database provider. // -// Currently the following database providers are supported: +// Currently, the following database providers are supported: // -// * Postgres -// * Sqlite -// . -func New(s *Setup) (Interface, error) { - // validate the setup being provided - // - // https://pkg.go.dev/github.com/go-vela/server/database?tab=doc#Setup.Validate - err := s.Validate() +// * postgres +// * sqlite3 +func New(c *Config) (Interface, error) { + // validate the configuration being provided + err := c.Validate() if err != nil { return nil, err } - logrus.Debug("creating database service from setup") + // create new database engine + e := &engine{ + Config: c, + Database: new(gorm.DB), + Logger: logrus.NewEntry(logrus.StandardLogger()).WithField("database", c.Driver), + } + + e.Logger.Trace("creating database engine from configuration") // process the database driver being provided - switch s.Driver { + switch c.Driver { case constants.DriverPostgres: - // handle the Postgres database driver being provided - // - // https://pkg.go.dev/github.com/go-vela/server/database?tab=doc#Setup.Postgres - return s.Postgres() + // create the new Postgres database client + e.Database, err = gorm.Open(postgres.Open(e.Config.Address), &gorm.Config{}) + if err != nil { + return nil, err + } case constants.DriverSqlite: - // handle the Sqlite database driver being provided - // - // https://pkg.go.dev/github.com/go-vela/server/database?tab=doc#Setup.Sqlite - return s.Sqlite() + // create the new Sqlite database client + e.Database, err = gorm.Open(sqlite.Open(e.Config.Address), &gorm.Config{}) + if err != nil { + return nil, err + } default: // handle an invalid database driver being provided - return nil, fmt.Errorf("invalid database driver provided: %s", s.Driver) + return nil, fmt.Errorf("invalid database driver provided: %s", c.Driver) } + + // capture database/sql database from gorm.io/gorm database + db, err := e.Database.DB() + if err != nil { + return nil, err + } + + // set the maximum amount of time a connection may be reused + db.SetConnMaxLifetime(e.Config.ConnectionLife) + // set the maximum number of connections in the idle connection pool + db.SetMaxIdleConns(e.Config.ConnectionIdle) + // set the maximum number of open connections to the database + db.SetMaxOpenConns(e.Config.ConnectionOpen) + + // verify connection to the database + err = e.Ping() + if err != nil { + return nil, err + } + + // create database agnostic engines for resources + err = e.NewResources() + if err != nil { + return nil, err + } + + return e, nil +} + +// NewTest creates and returns an engine that integrates with an in-memory database provider. +// +// This function is ONLY intended to be used for testing purposes. +func NewTest() (Interface, error) { + return New(&Config{ + Address: "file::memory:?cache=shared", + CompressionLevel: 3, + ConnectionLife: 30 * time.Minute, + ConnectionIdle: 2, + ConnectionOpen: 0, + Driver: "sqlite3", + EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", + SkipCreation: false, + }) } diff --git a/database/database_test.go b/database/database_test.go index 77de76d5b..6d18421bc 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -20,11 +20,13 @@ func TestDatabase_New(t *testing.T) { // setup tests tests := []struct { failure bool - setup *Setup + name string + config *Config }{ { + name: "failure with postgres", failure: true, - setup: &Setup{ + config: &Config{ Driver: "postgres", Address: "postgres://foo:bar@localhost:5432/vela", CompressionLevel: 3, @@ -36,8 +38,9 @@ func TestDatabase_New(t *testing.T) { }, }, { + name: "success with sqlite3", failure: false, - setup: &Setup{ + config: &Config{ Driver: "sqlite3", Address: "file::memory:?cache=shared", CompressionLevel: 3, @@ -49,10 +52,11 @@ func TestDatabase_New(t *testing.T) { }, }, { + name: "failure with invalid config", failure: true, - setup: &Setup{ - Driver: "mysql", - Address: "foo:bar@tcp(localhost:3306)/vela?charset=utf8mb4&parseTime=True&loc=Local", + config: &Config{ + Driver: "postgres", + Address: "", CompressionLevel: 3, ConnectionLife: 10 * time.Second, ConnectionIdle: 5, @@ -62,10 +66,11 @@ func TestDatabase_New(t *testing.T) { }, }, { + name: "failure with invalid driver", failure: true, - setup: &Setup{ - Driver: "postgres", - Address: "", + config: &Config{ + Driver: "mysql", + Address: "foo:bar@tcp(localhost:3306)/vela?charset=utf8mb4&parseTime=True&loc=Local", CompressionLevel: 3, ConnectionLife: 10 * time.Second, ConnectionIdle: 5, @@ -78,19 +83,21 @@ func TestDatabase_New(t *testing.T) { // run tests for _, test := range tests { - _, err := New(test.setup) + t.Run(test.name, func(t *testing.T) { + _, err := New(test.config) - if test.failure { - if err == nil { - t.Errorf("New should have returned err") - } + if test.failure { + if err == nil { + t.Errorf("New for %s should have returned err", test.name) + } - continue - } + return + } - if err != nil { - t.Errorf("New returned err: %v", err) - } + if err != nil { + t.Errorf("New for %s returned err: %v", test.name, err) + } + }) } } From 5b42b772c5c57ba8fed2116defc4cac7935a7066 Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Tue, 6 Jun 2023 08:02:54 -0500 Subject: [PATCH 09/13] feat(database): add FromCLIContext() --- database/context.go | 21 ++++- database/context_test.go | 177 ++++++++++++++++++++++++++------------- 2 files changed, 139 insertions(+), 59 deletions(-) diff --git a/database/context.go b/database/context.go index f3a872602..d27d73550 100644 --- a/database/context.go +++ b/database/context.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. @@ -6,6 +6,9 @@ package database import ( "context" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" ) const key = "database" @@ -35,3 +38,19 @@ func FromContext(c context.Context) Interface { func ToContext(c Setter, d Interface) { c.Set(key, d) } + +// FromCLIContext creates and returns a database engine from the urfave/cli context. +func FromCLIContext(c *cli.Context) (Interface, error) { + logrus.Debug("creating database engine from CLI configuration") + + return New(&Config{ + Address: c.String("database.addr"), + CompressionLevel: c.Int("database.compression.level"), + ConnectionLife: c.Duration("database.connection.life"), + ConnectionIdle: c.Int("database.connection.idle"), + ConnectionOpen: c.Int("database.connection.open"), + Driver: c.String("database.driver"), + EncryptionKey: c.String("database.encryption.key"), + SkipCreation: c.Bool("database.skip_creation"), + }) +} diff --git a/database/context_test.go b/database/context_test.go index 382567a43..4037b26c0 100644 --- a/database/context_test.go +++ b/database/context_test.go @@ -1,91 +1,152 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. package database import ( + "flag" + "reflect" "testing" + "time" "github.com/gin-gonic/gin" - "github.com/go-vela/server/database/sqlite" + "github.com/urfave/cli/v2" ) func TestDatabase_FromContext(t *testing.T) { - // setup types - want, _ := sqlite.NewTest() + _postgres, _ := testPostgres(t) + defer _postgres.Close() - defer func() { _sql, _ := want.Sqlite.DB(); _sql.Close() }() - - // setup context gin.SetMode(gin.TestMode) - context, _ := gin.CreateTestContext(nil) - context.Set(key, want) - - // run test - got := FromContext(context) - - if got != want { - t.Errorf("FromContext is %v, want %v", got, want) + ctx, _ := gin.CreateTestContext(nil) + ctx.Set(key, _postgres) + + typeCtx, _ := gin.CreateTestContext(nil) + typeCtx.Set(key, nil) + + nilCtx, _ := gin.CreateTestContext(nil) + nilCtx.Set(key, nil) + + // setup tests + tests := []struct { + name string + context *gin.Context + want Interface + }{ + { + name: "success", + context: ctx, + want: _postgres, + }, + { + name: "failure with nil", + context: nilCtx, + want: nil, + }, + { + name: "failure with wrong type", + context: typeCtx, + want: nil, + }, } -} -func TestDatabase_FromContext_Bad(t *testing.T) { - // setup context - gin.SetMode(gin.TestMode) - context, _ := gin.CreateTestContext(nil) - context.Set(key, nil) - - // run test - got := FromContext(context) - - if got != nil { - t.Errorf("FromContext is %v, want nil", got) + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := FromContext(test.context) + if !reflect.DeepEqual(got, test.want) { + t.Errorf("FromContext for %s is %v, want %v", test.name, got, test.want) + } + }) } } -func TestDatabase_FromContext_WrongType(t *testing.T) { - // setup context - gin.SetMode(gin.TestMode) +func TestDatabase_ToContext(t *testing.T) { context, _ := gin.CreateTestContext(nil) - context.Set(key, 1) - - // run test - got := FromContext(context) - if got != nil { - t.Errorf("FromContext is %v, want nil", got) + _postgres, _ := testPostgres(t) + defer _postgres.Close() + + _sqlite := testSqlite(t) + defer _sqlite.Close() + + // setup tests + tests := []struct { + name string + database *engine + want *engine + }{ + { + name: "success with postgres", + database: _postgres, + want: _postgres, + }, + { + name: "success with sqlite3", + database: _sqlite, + want: _sqlite, + }, } -} - -func TestDatabase_FromContext_Empty(t *testing.T) { - // setup context - gin.SetMode(gin.TestMode) - context, _ := gin.CreateTestContext(nil) - // run test - got := FromContext(context) + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ToContext(context, test.want) - if got != nil { - t.Errorf("FromContext is %v, want nil", got) + got := context.Value(key) + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToContext for %s is %v, want %v", test.name, got, test.want) + } + }) } } -func TestDatabase_ToContext(t *testing.T) { - // setup types - want, _ := sqlite.NewTest() +func TestDatabase_FromCLIContext(t *testing.T) { + flags := flag.NewFlagSet("test", 0) + flags.String("database.driver", "sqlite3", "doc") + flags.String("database.addr", "file::memory:?cache=shared", "doc") + flags.Int("database.compression.level", 3, "doc") + flags.Duration("database.connection.life", 10*time.Second, "doc") + flags.Int("database.connection.idle", 5, "doc") + flags.Int("database.connection.open", 20, "doc") + flags.String("database.encryption.key", "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", "doc") + flags.Bool("database.skip_creation", true, "doc") + + // setup tests + tests := []struct { + name string + failure bool + context *cli.Context + }{ + { + name: "success", + failure: false, + context: cli.NewContext(&cli.App{Name: "vela"}, flags, nil), + }, + { + name: "failure", + failure: true, + context: cli.NewContext(&cli.App{Name: "vela"}, flag.NewFlagSet("test", 0), nil), + }, + } - defer func() { _sql, _ := want.Sqlite.DB(); _sql.Close() }() + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := FromCLIContext(test.context) - // setup context - gin.SetMode(gin.TestMode) - context, _ := gin.CreateTestContext(nil) - ToContext(context, want) + if test.failure { + if err == nil { + t.Errorf("FromCLIContext for %s should have returned err", test.name) + } - // run test - got := context.Value(key) + return + } - if got != want { - t.Errorf("ToContext is %v, want %v", got, want) + if err != nil { + t.Errorf("FromCLIContext for %s returned err: %v", test.name, err) + } + }) } } From 6103860f356c51e2392c81d89531cffeede7babd Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Tue, 6 Jun 2023 08:03:14 -0500 Subject: [PATCH 10/13] chore: use database.FromCLIContext() --- cmd/vela-server/database.go | 35 ----------------------------------- cmd/vela-server/server.go | 3 ++- 2 files changed, 2 insertions(+), 36 deletions(-) delete mode 100644 cmd/vela-server/database.go diff --git a/cmd/vela-server/database.go b/cmd/vela-server/database.go deleted file mode 100644 index e4db992ec..000000000 --- a/cmd/vela-server/database.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package main - -import ( - "github.com/go-vela/server/database" - - "github.com/sirupsen/logrus" - - "github.com/urfave/cli/v2" -) - -// helper function to setup the database from the CLI arguments. -func setupDatabase(c *cli.Context) (database.Interface, error) { - logrus.Debug("Creating database client from CLI configuration") - - // database configuration - _setup := &database.Setup{ - Driver: c.String("database.driver"), - Address: c.String("database.addr"), - CompressionLevel: c.Int("database.compression.level"), - ConnectionLife: c.Duration("database.connection.life"), - ConnectionIdle: c.Int("database.connection.idle"), - ConnectionOpen: c.Int("database.connection.open"), - EncryptionKey: c.String("database.encryption.key"), - SkipCreation: c.Bool("database.skip_creation"), - } - - // setup the database - // - // https://pkg.go.dev/github.com/go-vela/server/database?tab=doc#New - return database.New(_setup) -} diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 80508e4b0..0d95e88e5 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -15,6 +15,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" "github.com/go-vela/server/router" "github.com/go-vela/server/router/middleware" "github.com/sirupsen/logrus" @@ -61,7 +62,7 @@ func server(c *cli.Context) error { return err } - database, err := setupDatabase(c) + database, err := database.FromCLIContext(c) if err != nil { return err } From 99f7a6c1e779dc77b9b982738c493e26c9fb51bd Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Tue, 6 Jun 2023 08:03:40 -0500 Subject: [PATCH 11/13] chore(database): remove unused code --- database/setup.go | 140 ----------------------- database/setup_test.go | 245 ----------------------------------------- 2 files changed, 385 deletions(-) delete mode 100644 database/setup.go delete mode 100644 database/setup_test.go diff --git a/database/setup.go b/database/setup.go deleted file mode 100644 index a4b553d4d..000000000 --- a/database/setup.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package database - -import ( - "fmt" - "strings" - "time" - - "github.com/go-vela/server/database/postgres" - "github.com/go-vela/server/database/sqlite" - "github.com/go-vela/types/constants" - "github.com/sirupsen/logrus" -) - -// Setup represents the configuration necessary for -// creating a Vela service capable of integrating -// with a configured database system. -type Setup struct { - // Database Configuration - - // specifies the driver to use for the database client - Driver string - // specifies the address to use for the database client - Address string - // specifies the level of compression to use for the database client - CompressionLevel int - // specifies the connection duration to use for the database client - ConnectionLife time.Duration - // specifies the maximum idle connections for the database client - ConnectionIdle int - // specifies the maximum open connections for the database client - ConnectionOpen int - // specifies the encryption key to use for the database client - EncryptionKey string - // specifies to skip creating tables and indexes for the database client - SkipCreation bool -} - -// Postgres creates and returns a Vela service capable of -// integrating with a Postgres database system. -func (s *Setup) Postgres() (Interface, error) { - logrus.Trace("creating postgres database client from setup") - - // create new Postgres database service - // - // https://pkg.go.dev/github.com/go-vela/server/database/postgres?tab=doc#New - return postgres.New( - postgres.WithAddress(s.Address), - postgres.WithCompressionLevel(s.CompressionLevel), - postgres.WithConnectionLife(s.ConnectionLife), - postgres.WithConnectionIdle(s.ConnectionIdle), - postgres.WithConnectionOpen(s.ConnectionOpen), - postgres.WithEncryptionKey(s.EncryptionKey), - postgres.WithSkipCreation(s.SkipCreation), - ) -} - -// Sqlite creates and returns a Vela service capable of -// integrating with a Sqlite database system. -func (s *Setup) Sqlite() (Interface, error) { - logrus.Trace("creating sqlite database client from setup") - - // create new Sqlite database service - // - // https://pkg.go.dev/github.com/go-vela/server/database/sqlite?tab=doc#New - return sqlite.New( - sqlite.WithAddress(s.Address), - sqlite.WithCompressionLevel(s.CompressionLevel), - sqlite.WithConnectionLife(s.ConnectionLife), - sqlite.WithConnectionIdle(s.ConnectionIdle), - sqlite.WithConnectionOpen(s.ConnectionOpen), - sqlite.WithEncryptionKey(s.EncryptionKey), - sqlite.WithSkipCreation(s.SkipCreation), - ) -} - -// Validate verifies the necessary fields for the -// provided configuration are populated correctly. -func (s *Setup) Validate() error { - logrus.Trace("validating database setup for client") - - // verify a database driver was provided - if len(s.Driver) == 0 { - return fmt.Errorf("no database driver provided") - } - - // verify a database address was provided - if len(s.Address) == 0 { - return fmt.Errorf("no database address provided") - } - - // check if the database address has a trailing slash - if strings.HasSuffix(s.Address, "/") { - return fmt.Errorf("database address must not have trailing slash") - } - - // verify a database encryption key was provided - if len(s.EncryptionKey) == 0 { - return fmt.Errorf("no database encryption key provided") - } - - // verify the database compression level is valid - switch s.CompressionLevel { - case constants.CompressionNegOne: - fallthrough - case constants.CompressionZero: - fallthrough - case constants.CompressionOne: - fallthrough - case constants.CompressionTwo: - fallthrough - case constants.CompressionThree: - fallthrough - case constants.CompressionFour: - fallthrough - case constants.CompressionFive: - fallthrough - case constants.CompressionSix: - fallthrough - case constants.CompressionSeven: - fallthrough - case constants.CompressionEight: - fallthrough - case constants.CompressionNine: - break - default: - return fmt.Errorf("database compression level must be between %d and %d - provided level: %d", constants.CompressionNegOne, constants.CompressionNine, s.CompressionLevel) - } - - // enforce AES-256 for the encryption key - explicitly check for 32 characters in the key - if len(s.EncryptionKey) != 32 { - return fmt.Errorf("database encryption key must have 32 characters - provided length: %d", len(s.EncryptionKey)) - } - - // setup is valid - return nil -} diff --git a/database/setup_test.go b/database/setup_test.go deleted file mode 100644 index 4ec2c958c..000000000 --- a/database/setup_test.go +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package database - -import ( - "testing" - "time" -) - -func TestDatabase_Setup_Postgres(t *testing.T) { - // setup types - _setup := &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - } - - // setup tests - tests := []struct { - failure bool - setup *Setup - }{ - { - failure: true, - setup: _setup, - }, - { - failure: true, - setup: &Setup{Driver: "postgres"}, - }, - } - - // run tests - for _, test := range tests { - _, err := test.setup.Postgres() - - if test.failure { - if err == nil { - t.Errorf("Postgres should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("Postgres returned err: %v", err) - } - } -} - -func TestDatabase_Setup_Sqlite(t *testing.T) { - // setup types - _setup := &Setup{ - Driver: "sqlite3", - Address: "file::memory:?cache=shared", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - } - - // setup tests - tests := []struct { - failure bool - setup *Setup - }{ - { - failure: false, - setup: _setup, - }, - { - failure: true, - setup: &Setup{Driver: "sqlite3"}, - }, - } - - // run tests - for _, test := range tests { - _, err := test.setup.Sqlite() - - if test.failure { - if err == nil { - t.Errorf("Sqlite should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("Sqlite returned err: %v", err) - } - } -} - -func TestDatabase_Setup_Validate(t *testing.T) { - // setup tests - tests := []struct { - failure bool - setup *Setup - }{ - { - failure: false, - setup: &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - { - failure: false, - setup: &Setup{ - Driver: "sqlite3", - Address: "file::memory:?cache=shared", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - { - failure: false, - setup: &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: -1, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - { - failure: true, - setup: &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela/", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - { - failure: true, - setup: &Setup{ - Driver: "", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - { - failure: true, - setup: &Setup{ - Driver: "postgres", - Address: "", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - { - failure: true, - setup: &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "", - SkipCreation: false, - }, - }, - { - failure: true, - setup: &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: 3, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0", - SkipCreation: false, - }, - }, - { - failure: true, - setup: &Setup{ - Driver: "postgres", - Address: "postgres://foo:bar@localhost:5432/vela", - CompressionLevel: 10, - ConnectionLife: 10 * time.Second, - ConnectionIdle: 5, - ConnectionOpen: 20, - EncryptionKey: "A1B2C3D4E5G6H7I8J9K0LMNOPQRSTUVW", - SkipCreation: false, - }, - }, - } - - // run tests - for _, test := range tests { - err := test.setup.Validate() - - if test.failure { - if err == nil { - t.Errorf("Validate should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("Validate returned err: %v", err) - } - } -} From 7a8760960e5f33be9bc82b4f8ed640bd24e02f37 Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Tue, 6 Jun 2023 08:03:57 -0500 Subject: [PATCH 12/13] chore: misc updates --- database/flags.go | 9 ++------- database/interface.go | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/database/flags.go b/database/flags.go index 75d3c109b..f8d4fd36e 100644 --- a/database/flags.go +++ b/database/flags.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. @@ -11,13 +11,8 @@ import ( "github.com/urfave/cli/v2" ) -// Flags represents all supported command line -// interface (CLI) flags for the database. -// -// https://pkg.go.dev/github.com/urfave/cli?tab=doc#Flag +// Flags represents all supported command line interface (CLI) flags for the database. var Flags = []cli.Flag{ - // Database Flags - &cli.StringFlag{ EnvVars: []string{"VELA_DATABASE_DRIVER", "DATABASE_DRIVER"}, FilePath: "/vela/database/driver", diff --git a/database/interface.go b/database/interface.go index 975206c93..eed4b27c8 100644 --- a/database/interface.go +++ b/database/interface.go @@ -18,15 +18,23 @@ import ( "github.com/go-vela/server/database/worker" ) -// Interface represents the interface for Vela integrating -// with the different supported Database backends. +// Interface represents the interface for integrating with the supported database providers. type Interface interface { - // Database Interface Functions + // Generic Interface Functions - // Driver defines a function that outputs - // the configured database driver. + // TODO: Add this function to the interface once other code has been updated to use the agnostic engine. + // + // Close defines a function that stops and terminates the connection to the database. + // Close() error + + // Driver defines a function that outputs the configured database driver. Driver() string + // Ping defines a function that sends a "ping" request to the configured database. + Ping() error + + // Resource Interface Functions + // BuildInterface defines the interface for builds stored in the database. build.BuildInterface From 25c289366a882e0ccc000dfaa36f20847539af1c Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Wed, 7 Jun 2023 11:47:33 -0500 Subject: [PATCH 13/13] chore: fix typos --- database/database.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/database/database.go b/database/database.go index d60cb5189..5fe3de951 100644 --- a/database/database.go +++ b/database/database.go @@ -30,21 +30,21 @@ import ( type ( // Config represents the settings required to create the engine that implements the Interface. Config struct { - // specifies the address to use for the database client + // specifies the address to use for the database engine Address string - // specifies the level of compression to use for the database client + // specifies the level of compression to use for the database engine CompressionLevel int - // specifies the connection duration to use for the database client + // specifies the connection duration to use for the database engine ConnectionLife time.Duration - // specifies the maximum idle connections for the database client + // specifies the maximum idle connections for the database engine ConnectionIdle int - // specifies the maximum open connections for the database client + // specifies the maximum open connections for the database engine ConnectionOpen int - // specifies the driver to use for the database client + // specifies the driver to use for the database engine Driver string - // specifies the encryption key to use for the database client + // specifies the encryption key to use for the database engine EncryptionKey string - // specifies to skip creating tables and indexes for the database client + // specifies to skip creating tables and indexes for the database engine SkipCreation bool }