diff --git a/api/queryRunner/queryrunner.go b/api/queryRunner/queryrunner.go index 2e3fa28d..af78c9c4 100644 --- a/api/queryRunner/queryrunner.go +++ b/api/queryRunner/queryrunner.go @@ -31,7 +31,7 @@ type DocumentQueryEvent struct { ID uuid.UUID `json:"id"` } -func (s QueryRunner) Process(ctx context.Context, req *types.Message) error { +func (s *QueryRunner) Process(ctx context.Context, req *types.Message) error { var body query.Document err := json.Unmarshal([]byte(*req.Body), &body) if err != nil { diff --git a/api/queryRunner/queryrunner_test.go b/api/queryRunner/queryrunner_test.go index 81118507..3f38da28 100644 --- a/api/queryRunner/queryrunner_test.go +++ b/api/queryRunner/queryrunner_test.go @@ -10,6 +10,7 @@ import ( "queryorchestration/internal/job/collector" "queryorchestration/internal/query" "queryorchestration/internal/query/result" + "queryorchestration/internal/serviceconfig" "testing" "github.com/aws/aws-sdk-go-v2/service/sqs/types" @@ -26,16 +27,15 @@ func TestQueryRunner(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := query.New(db, &query.Services{ - Result: result.New(db), + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := query.New(cfg, &query.Services{ + Result: result.New(cfg), Text: documenttext.New(), - Collector: collector.New(db, &collector.Services{}), + Collector: collector.New(cfg, &collector.Services{}), }) runner := controllers.NewQueryRunner(validator.New(), &controllers.Services{ diff --git a/api/queryService/client_test.go b/api/queryService/client_test.go index 42880ccd..0014d370 100644 --- a/api/queryService/client_test.go +++ b/api/queryService/client_test.go @@ -9,6 +9,7 @@ import ( "queryorchestration/internal/client" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" "strings" "testing" @@ -24,14 +25,13 @@ func TestCreateClient(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Client: client.New(db), + Client: client.New(cfg), }) body := queryservice.ClientCreate{ @@ -64,14 +64,13 @@ func TestGetClient(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Client: client.New(db), + Client: client.New(cfg), }) e := echo.New() @@ -107,14 +106,13 @@ func TestUpdateClient(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Client: client.New(db), + Client: client.New(cfg), }) cs := false diff --git a/api/queryService/job_test.go b/api/queryService/job_test.go index 6d4519f3..808a8dba 100644 --- a/api/queryService/job_test.go +++ b/api/queryService/job_test.go @@ -13,6 +13,7 @@ import ( documenttext "queryorchestration/internal/document/text" "queryorchestration/internal/job" "queryorchestration/internal/job/collector" + "queryorchestration/internal/serviceconfig" "strings" "testing" @@ -29,16 +30,15 @@ func TestCreateJob(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) extract := documenttext.New() cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Job: job.New(db, &job.Services{ - Collector: collector.New(db, &collector.Services{ + Job: job.New(cfg, &job.Services{ + Collector: collector.New(cfg, &collector.Services{ Clean: documentclean.New(), Text: extract, }), @@ -81,15 +81,13 @@ func TestGetJob(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Job: job.New(db, &job.Services{ - Client: client.New(db), + Job: job.New(cfg, &job.Services{ + Client: client.New(cfg), }), }) @@ -133,15 +131,13 @@ func TestUpdateJob(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Job: job.New(db, &job.Services{ - Client: client.New(db), + Job: job.New(cfg, &job.Services{ + Client: client.New(cfg), }), }) diff --git a/api/queryService/jobcollector_test.go b/api/queryService/jobcollector_test.go index 8e25b474..670967c2 100644 --- a/api/queryService/jobcollector_test.go +++ b/api/queryService/jobcollector_test.go @@ -10,6 +10,7 @@ import ( documentclean "queryorchestration/internal/document/clean" documenttext "queryorchestration/internal/document/text" "queryorchestration/internal/job/collector" + "queryorchestration/internal/serviceconfig" "strings" "testing" @@ -26,11 +27,9 @@ func TestUpdateJobCollector(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) av := int32(2) body := queryservice.JobCollectorUpdate{ @@ -47,7 +46,7 @@ func TestUpdateJobCollector(t *testing.T) { extract := documenttext.New() cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Collector: collector.New(db, &collector.Services{ + Collector: collector.New(cfg, &collector.Services{ Clean: documentclean.New(), Text: extract, }), @@ -83,11 +82,9 @@ func TestGetJobCollectorByJobId(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) e := echo.New() req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("")) @@ -95,7 +92,7 @@ func TestGetJobCollectorByJobId(t *testing.T) { ctx := e.NewContext(req, rec) extract := documenttext.New() - svc := collector.New(db, &collector.Services{ + svc := collector.New(cfg, &collector.Services{ Clean: documentclean.New(), Text: extract, }) diff --git a/api/queryService/query_test.go b/api/queryService/query_test.go index 31b080c0..fe71df3d 100644 --- a/api/queryService/query_test.go +++ b/api/queryService/query_test.go @@ -14,6 +14,7 @@ import ( "queryorchestration/internal/job/collector" "queryorchestration/internal/query" "queryorchestration/internal/query/result" + "queryorchestration/internal/serviceconfig" "strings" "testing" @@ -31,14 +32,12 @@ func TestCreateQuery(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Query: query.New(db, &query.Services{}), + Query: query.New(cfg, &query.Services{}), }) body := queryservice.QueryCreate{ @@ -73,14 +72,12 @@ func TestListQueries(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Query: query.New(db, &query.Services{}), + Query: query.New(cfg, &query.Services{}), }) e := echo.New() @@ -118,14 +115,12 @@ func TestGetQuery(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Query: query.New(db, &query.Services{}), + Query: query.New(cfg, &query.Services{}), }) e := echo.New() @@ -162,14 +157,12 @@ func TestUpdateQuery(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Query: query.New(db, &query.Services{}), + Query: query.New(cfg, &query.Services{}), }) av := int32(2) @@ -210,23 +203,21 @@ func TestTestQuery(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - docsvc := document.New(db) - col := collector.New(db, &collector.Services{ + docsvc := document.New(cfg) + col := collector.New(cfg, &collector.Services{ Clean: documentclean.New(), Text: documenttext.New(), }) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ Collector: col, - Query: query.New(db, &query.Services{ + Query: query.New(cfg, &query.Services{ Document: docsvc, Collector: col, - Result: result.New(db), + Result: result.New(cfg), }), }) diff --git a/build/Dockerfile b/build/Dockerfile index c58ce466..584aa899 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -19,7 +19,9 @@ RUN --mount=type=cache,target=/go/pkg/mod/ \ FROM scratch AS final -WORKDIR /app +ENV PWD=/app + +WORKDIR ${PWD} COPY database/migrations/ database/migrations/ diff --git a/cmd/queryRunner/main.go b/cmd/queryRunner/main.go index dac3d2ec..69501234 100644 --- a/cmd/queryRunner/main.go +++ b/cmd/queryRunner/main.go @@ -2,14 +2,17 @@ package main import ( "context" + "log/slog" + "os" controllers "queryorchestration/api/queryRunner" + "queryorchestration/internal/document" documentclean "queryorchestration/internal/document/clean" documenttext "queryorchestration/internal/document/text" "queryorchestration/internal/job/collector" "queryorchestration/internal/query" "queryorchestration/internal/query/result" - "queryorchestration/internal/server" - "queryorchestration/internal/server/queue" + "queryorchestration/internal/server/runner" + "queryorchestration/internal/serviceconfig" _ "github.com/lib/pq" ) @@ -17,29 +20,41 @@ import ( func main() { ctx := context.Background() - queryrunner := func(cfg *server.Config) queue.Controller { - extract := documenttext.New() + cfg := &runner.BaseConfig{} + + if err := serviceconfig.InitializeConfig(cfg); err != nil { + slog.Error("Error initializing config", "err", err) + os.Exit(1) + } + + cfg.ControllerFunc = func() runner.Controller { text := documenttext.New() clean := documentclean.New() - coll := collector.New(cfg.Database, &collector.Services{ + coll := collector.New(cfg, &collector.Services{ Text: text, Clean: clean, }) - res := result.New(cfg.Database) - svc := query.New(cfg.Database, &query.Services{ + res := result.New(cfg) + doc := document.New(cfg) + svc := query.New(cfg, &query.Services{ Result: res, - Text: extract, + Text: text, Collector: coll, + Document: doc, }) - return controllers.NewQueryRunner(cfg.Validator, &controllers.Services{ + c := controllers.NewQueryRunner(cfg.GetValidator(), &controllers.Services{ Query: svc, }) + + return &c } - server := queue.NewServer(ctx, &queue.ListenerConfig{ - Controller: queryrunner, - }) + server, err := runner.New(ctx, cfg) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } server.Listen(ctx) } diff --git a/cmd/queryService/main.go b/cmd/queryService/main.go index e15cab45..19b0527a 100644 --- a/cmd/queryService/main.go +++ b/cmd/queryService/main.go @@ -2,9 +2,12 @@ package main import ( "context" - "log" + "fmt" + "log/slog" + "os" queryservice "queryorchestration/api/queryService" "queryorchestration/internal/client" + "queryorchestration/internal/document" documentclean "queryorchestration/internal/document/clean" documenttext "queryorchestration/internal/document/text" "queryorchestration/internal/export" @@ -12,32 +15,41 @@ import ( "queryorchestration/internal/job/collector" "queryorchestration/internal/query" "queryorchestration/internal/query/result" - "queryorchestration/internal/server" - "queryorchestration/internal/server/api" + service "queryorchestration/internal/server/service" + "queryorchestration/internal/serviceconfig" - "github.com/labstack/echo/v4" + "github.com/getkin/kin-openapi/openapi3" _ "github.com/lib/pq" ) func main() { ctx := context.Background() - registerHandlers := func(cfg *server.Config, e *echo.Echo) *api.APIConfig { - exp := export.New(cfg.Database) + cfg := &service.BaseConfig{} + + if err := serviceconfig.InitializeConfig(cfg); err != nil { + fmt.Printf("Error initializing config: %s", err) + os.Exit(1) + } + + cfg.RegisterHandlersFunc = func() (*openapi3.T, error) { + exp := export.New() extract := documenttext.New() clean := documentclean.New() - col := collector.New(cfg.Database, &collector.Services{ + res := result.New(cfg) + col := collector.New(cfg, &collector.Services{ Text: extract, Clean: clean, }) - res := result.New(cfg.Database) - que := query.New(cfg.Database, &query.Services{ + doc := document.New(cfg) + que := query.New(cfg, &query.Services{ Text: extract, Result: res, Collector: col, + Document: doc, }) - cli := client.New(cfg.Database) - jbb := job.New(cfg.Database, &job.Services{ + cli := client.New(cfg) + jbb := job.New(cfg, &job.Services{ Collector: col, Client: cli, }) @@ -50,23 +62,23 @@ func main() { Job: jbb, } - cons := queryservice.NewControllers(cfg.Validator, services) + cons := queryservice.NewControllers(cfg.GetValidator(), services) - queryservice.RegisterHandlersWithBaseURL(e, cons, "") + queryservice.RegisterHandlersWithBaseURL(cfg.Router, cons, "") swagger, err := queryservice.GetSwagger() if err != nil { - log.Panicf("Error loading swagger: %s", err) + return nil, fmt.Errorf("error loading swagger: %s", err) } - return &api.APIConfig{ - Swagger: swagger, - } + return swagger, nil } - server := api.New(ctx, &api.Config{ - RegisterHandlers: registerHandlers, - }) + server, err := service.New(ctx, cfg) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } server.Listen() } diff --git a/cmd/testServiceInit/main.go b/cmd/testServiceInit/main.go deleted file mode 100644 index 5659422d..00000000 --- a/cmd/testServiceInit/main.go +++ /dev/null @@ -1,36 +0,0 @@ -package main - -import ( - "fmt" - "os" - "queryorchestration/internal/serviceconfig" -) - -// Note: All that a service needs to have its own config is to create a new -// config type (any name you want) and add your customer variables to it that -// then call the serviceconfig.InitializeConfig() function on it. -type AnyServiceNameConfig struct { - // BaseConfig has all of the common base configuration that all services should have - // we can adjust what is in there as needed. - serviceconfig.BaseConfig // Embed the base configuration - - // add any custom values that this service needs - AppEnv string `env:"APP_ENV"` -} - -// main -// Dead simple example of of how to initialize a service custom config - AnyServiceNameConfig object in any project code. -func main() { - - // create your own customer service config - cfg := &AnyServiceNameConfig{} - // and let the shared code initialize it for you correctly. - if err := serviceconfig.InitializeConfig(cfg); err != nil { - fmt.Printf("Error initializing the custom config (see output): %v\n", err) - os.Exit(1) - } - - fmt.Printf("config: %+v\n", cfg) - cfg.Logger.Info("Service initialized ok.") - -} diff --git a/devbox.json b/devbox.json index f5d130d6..c2da67df 100644 --- a/devbox.json +++ b/devbox.json @@ -30,7 +30,7 @@ "DB_HOST": "localhost", "DB_PORT": "5432", "DB_NAME": "query_orchestration", - "DB_NOSSL": "1" + "DB_NOSSL": "true" }, "env_from": ".env" } diff --git a/docs/database.md b/docs/database.md index dc5f03d6..a1aa9ee4 100644 --- a/docs/database.md +++ b/docs/database.md @@ -6,6 +6,11 @@ You can also edit live with https://mermaid.live This diagram should be updated manually when there are changes to the database schema. +To generate the diagram directly use the following command: +```bash +docker run --rm -v $(pwd):/data minlag/mermaid-cli -i /data/diagram.mmd -o /data/output.png -w 1024 -H 768 +``` + ```mermaid erDiagram clients ||--o{ jobs : has diff --git a/internal/client/create.go b/internal/client/create.go index 90983249..e8e500cc 100644 --- a/internal/client/create.go +++ b/internal/client/create.go @@ -13,7 +13,7 @@ func (s *Service) Create(ctx context.Context, name string) (uuid.UUID, error) { return uuid.Nil, err } - id, err := s.db.Queries.CreateClient(ctx, name) + id, err := s.cfg.GetDBQueries().CreateClient(ctx, name) if err != nil { return uuid.Nil, err } diff --git a/internal/client/create_test.go b/internal/client/create_test.go index 0370c4a8..cb7154f5 100644 --- a/internal/client/create_test.go +++ b/internal/client/create_test.go @@ -5,6 +5,7 @@ import ( "queryorchestration/internal/client" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -19,13 +20,11 @@ func TestCreate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := client.New(db) + svc := client.New(cfg) name := "client_name" aid := uuid.New() diff --git a/internal/client/get.go b/internal/client/get.go index 50ff237d..a2ff900f 100644 --- a/internal/client/get.go +++ b/internal/client/get.go @@ -8,7 +8,7 @@ import ( ) func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Client, error) { - client, err := s.db.Queries.GetClient(ctx, database.MustToDBUUID(id)) + client, err := s.cfg.GetDBQueries().GetClient(ctx, database.MustToDBUUID(id)) if err != nil { return nil, err } diff --git a/internal/client/get_test.go b/internal/client/get_test.go index f1567f11..cf2a208f 100644 --- a/internal/client/get_test.go +++ b/internal/client/get_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/client" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -20,13 +21,11 @@ func TestGet(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := client.New(db) + svc := client.New(cfg) id := uuid.New() diff --git a/internal/client/service.go b/internal/client/service.go index 84c1c02a..87cbd58f 100644 --- a/internal/client/service.go +++ b/internal/client/service.go @@ -3,7 +3,7 @@ package client import ( "errors" "fmt" - "queryorchestration/internal/database" + "queryorchestration/internal/serviceconfig" "regexp" "strings" @@ -70,11 +70,11 @@ func normalizeName(name *string) error { } type Service struct { - db *database.Connection + cfg serviceconfig.ConfigProvider } -func New(db *database.Connection) *Service { +func New(cfg serviceconfig.ConfigProvider) *Service { return &Service{ - db, + cfg, } } diff --git a/internal/client/service_test.go b/internal/client/service_test.go index 5a9ea9ff..c08f2165 100644 --- a/internal/client/service_test.go +++ b/internal/client/service_test.go @@ -2,8 +2,8 @@ package client_test import ( "queryorchestration/internal/client" - "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" "testing" "github.com/pashagolub/pgxmock/v3" @@ -15,12 +15,10 @@ func TestService(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := client.New(db) + svc := client.New(cfg) assert.NotNil(t, svc) } diff --git a/internal/client/update.go b/internal/client/update.go index 1018b131..2ac03511 100644 --- a/internal/client/update.go +++ b/internal/client/update.go @@ -35,7 +35,7 @@ func (s *Service) Update(ctx context.Context, entity *Update) error { } func (s *Service) submitUpdate(ctx context.Context, entity *Update) error { - return database.ExecuteTransaction(ctx, s.db, func(ctx context.Context, q *repository.Queries) error { + return s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error { id := database.MustToDBUUID(entity.ID) if entity.Name != nil { diff --git a/internal/client/update_test.go b/internal/client/update_test.go index 84d153f5..cd7f3fe4 100644 --- a/internal/client/update_test.go +++ b/internal/client/update_test.go @@ -4,6 +4,7 @@ import ( "context" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -18,13 +19,11 @@ func TestUpdate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := New(db) + svc := New(cfg) c := Client{ ID: uuid.New(), @@ -119,13 +118,11 @@ func TestSubmitUpdate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := New(db) + svc := New(cfg) c := Client{ ID: uuid.New(), diff --git a/internal/database/config_test.go b/internal/database/config_test.go deleted file mode 100644 index 8af2673c..00000000 --- a/internal/database/config_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package database - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestMustGetURIInvalidPort(t *testing.T) { - t.Setenv("DB_USER", "user") - t.Setenv("DB_PASS", "pass") - t.Setenv("DB_HOST", "host") - t.Setenv("DB_PORT", "port") - t.Setenv("DB_NAME", "name") - - assert.Panics(t, func() { mustGetURI() }) -} - -func TestMustGetURINoSSL(t *testing.T) { - t.Setenv("DB_USER", "user") - t.Setenv("DB_PASS", "pass") - t.Setenv("DB_HOST", "host") - t.Setenv("DB_PORT", "5432") - t.Setenv("DB_NAME", "name") - t.Setenv("DB_NOSSL", "1") - - connStr := mustGetURI() - assert.Equal(t, "postgres://user:pass@host:5432/name?sslmode=disable", connStr.String()) -} - -func TestMustGetURINoEnv(t *testing.T) { - t.Setenv("DB_USER", "") - t.Setenv("DB_PASS", "") - t.Setenv("DB_HOST", "") - t.Setenv("DB_PORT", "") - t.Setenv("DB_NAME", "") - - assert.Panics(t, func() { mustGetURI() }) -} - -func TestGetPoolConfig(t *testing.T) { - t.Setenv("DB_USER", "user") - t.Setenv("DB_PASS", "pass") - t.Setenv("DB_HOST", "host") - t.Setenv("DB_PORT", "5432") - t.Setenv("DB_NAME", "name") - - config, err := getPoolConfig() - assert.Nil(t, err) - assert.Equal(t, config.ConnString(), mustGetURI().String()) -} - -func TestMustGetPoolConfig(t *testing.T) { - t.Setenv("DB_USER", "user") - t.Setenv("DB_PASS", "pass") - t.Setenv("DB_HOST", "host") - t.Setenv("DB_PORT", "5432") - t.Setenv("DB_NAME", "name") - - config := mustGetPoolConfig() - assert.Equal(t, config.ConnString(), mustGetURI().String()) -} - -func TestMustGetPoolConfigInvalidPort(t *testing.T) { - t.Setenv("DB_USER", "user") - t.Setenv("DB_PASS", "pass") - t.Setenv("DB_HOST", "host") - t.Setenv("DB_PORT", "invalid_port") - t.Setenv("DB_NAME", "name") - - assert.Panics(t, func() { mustGetPoolConfig() }) -} diff --git a/internal/database/connection.go b/internal/database/connection.go deleted file mode 100644 index fd3ae4d5..00000000 --- a/internal/database/connection.go +++ /dev/null @@ -1,135 +0,0 @@ -package database - -import ( - "context" - "fmt" - "log" - "net/url" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/server/env" - - "github.com/docker/go-connections/nat" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgxpool" -) - -type config struct { - user string - password string - host string - port nat.Port - name string - disableSSL bool -} - -func mustGetConfig() *config { - user := env.GetPanic("DB_USER") - pass := env.GetPanic("DB_PASS") - host := env.GetPanic("DB_HOST") - port := env.GetPanic("DB_PORT") - natPort, err := nat.NewPort("tcp", port) - if err != nil { - log.Panicf("Failed to create port: %v", err) - } - name := env.GetPanic("DB_NAME") - - disableSSL, _ := env.Get("DB_NOSSL") - - return &config{ - user: user, - password: pass, - host: host, - port: natPort, - name: name, - disableSSL: disableSSL == "1", - } -} - -func mustGetURI() *url.URL { - driver := "postgres" - conf := mustGetConfig() - opts := "" - - if conf.disableSSL { - opts += "sslmode=disable" - } - - connStr := fmt.Sprintf("%s://%s:%s@%s:%d/%s?%s", driver, conf.user, conf.password, conf.host, conf.port.Int(), conf.name, opts) - - uri, err := url.Parse(connStr) - if err != nil { - log.Panicf("Unable to parse URI: %s", err) - } - - return uri -} - -func getPoolConfig() (*pgxpool.Config, error) { - connStr := mustGetURI() - - config, err := pgxpool.ParseConfig(connStr.String()) - if err != nil { - return nil, err - } - - enumTypes := []string{ - "querytype", - } - - config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error { - for _, enumName := range enumTypes { - dt, err := conn.LoadType(ctx, enumName) - if err != nil { - return fmt.Errorf("failed to load suitable enum type: %w", err) - } - conn.TypeMap().RegisterType(dt) - } - return nil - } - - return config, nil -} - -func mustGetPoolConfig() *pgxpool.Config { - config, err := getPoolConfig() - if err != nil { - log.Panicf("Unable to create database config: %v\n", err) - } - - return config -} - -func GetDBPool(ctx context.Context) *pgxpool.Pool { - config := mustGetPoolConfig() - - pool, err := pgxpool.NewWithConfig(ctx, config) - if err != nil { - log.Panicf("Unable to create database pool: %v\n", err) - } - - return pool -} - -func ExecuteTransaction(ctx context.Context, db *Connection, executeQueries func(context.Context, *repository.Queries) error) error { - tx, err := db.Pool.Begin(ctx) - if err != nil { - return err - } - defer func() { - _ = tx.Rollback(ctx) - }() - - qtx := db.Queries.WithTx(tx) - - err = executeQueries(ctx, qtx) - if err != nil { - return err - } - - err = tx.Commit(ctx) - if err != nil { - return err - } - - return nil -} diff --git a/internal/database/migrations.go b/internal/database/migrations.go deleted file mode 100644 index 90021813..00000000 --- a/internal/database/migrations.go +++ /dev/null @@ -1,81 +0,0 @@ -package database - -import ( - "context" - "database/sql" - "fmt" - "log" - "os" - "path" - - "github.com/golang-migrate/migrate/v4" - _ "github.com/golang-migrate/migrate/v4/database/postgres" - _ "github.com/golang-migrate/migrate/v4/source/file" - _ "github.com/lib/pq" -) - -func createDB() { - driver := "postgres" - conf := mustGetConfig() - - opts := "" - if conf.disableSSL { - opts += "sslmode=disable" - } - - connStr := fmt.Sprintf("%s://%s:%s@%s:%d/?%s", driver, conf.user, conf.password, conf.host, conf.port.Int(), opts) - - db, err := sql.Open("postgres", connStr) - if err != nil { - log.Panicf("Error opening database: %v", err) - } - - err = db.Ping() - if err != nil { - log.Panicf("Error pinging database: %v", err) - } - - rs, err := db.Query(fmt.Sprintf("SELECT 'CREATE DATABASE %s' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '%s')", conf.name, conf.name)) - if err != nil { - log.Panicf("Error creating database: %v", err) - } - - if rs.Next() { - log.Printf("Database created: %s", conf.name) - } else { - log.Printf("Database already exists: %s", conf.name) - } -} - -type MigrationConfig struct { - BasePath string -} - -func RunMigrations(ctx context.Context, config *MigrationConfig) { - createDB() - - connStr := mustGetURI() - - basePath := config.BasePath - if basePath == "" { - basePath = os.Getenv("PWD") - } - - migPath := "file://" + path.Join(basePath, "database/migrations") - m, err := migrate.New( - migPath, - connStr.String(), - ) - if err != nil { - log.Panicf("failed to create migrate instance: %v", err) - } - - err = m.Up() - if err == migrate.ErrNoChange { - log.Println("No changes required") - } else if err != nil { - log.Panicf("failed to apply migrations: %v", err) - } else { - log.Println("Migrations applied successfully!") - } -} diff --git a/internal/database/migrations/migrations.go b/internal/database/migrations/migrations.go new file mode 100644 index 00000000..9ac8f886 --- /dev/null +++ b/internal/database/migrations/migrations.go @@ -0,0 +1,69 @@ +package migrations + +import ( + "context" + "database/sql" + "fmt" + "log/slog" + "path" + "queryorchestration/internal/serviceconfig" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/postgres" + _ "github.com/golang-migrate/migrate/v4/source/file" + _ "github.com/lib/pq" +) + +func createDB(cfg serviceconfig.ConfigProvider) error { + connStr := fmt.Sprintf("%s?%s", cfg.GetDBBaseURI(), cfg.GetDBOptsString()) + + db, err := sql.Open(cfg.GetDBDriver(), connStr) + if err != nil { + return fmt.Errorf("error opening database: %v", err) + } + + err = db.Ping() + if err != nil { + return fmt.Errorf("error pinging database: %v", err) + } + + rs, err := db.Query(fmt.Sprintf("SELECT 'CREATE DATABASE %s' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '%s')", cfg.GetDBName(), cfg.GetDBName())) + if err != nil { + return fmt.Errorf("error creating database: %v", err) + } + + if !rs.Next() { + slog.Info("database created", "name", cfg.GetDBName()) + } else { + slog.Info("database already exists", "name", cfg.GetDBName()) + } + + return nil +} + +func Run(ctx context.Context, cfg serviceconfig.ConfigProvider) error { + err := createDB(cfg) + if err != nil { + return err + } + + migPath := "file://" + path.Join(cfg.GetBasePath(), "database/migrations") + m, err := migrate.New( + migPath, + cfg.GetDBURI(), + ) + if err != nil { + return fmt.Errorf("failed to create migrate instance: %v", err) + } + + err = m.Up() + if err == migrate.ErrNoChange { + slog.Info("No migration changes required") + } else if err != nil { + return fmt.Errorf("failed to apply migrations: %v", err) + } else { + slog.Info("Migrations applied successfully!") + } + + return nil +} diff --git a/internal/database/migrations_test.go b/internal/database/migrations/migrations_test.go similarity index 50% rename from internal/database/migrations_test.go rename to internal/database/migrations/migrations_test.go index b496b2f8..db78e195 100644 --- a/internal/database/migrations_test.go +++ b/internal/database/migrations/migrations_test.go @@ -1,10 +1,11 @@ -package database_test +package migrations_test import ( "context" "os" "path" - "queryorchestration/internal/database" + "queryorchestration/internal/database/migrations" + "queryorchestration/internal/serviceconfig" "queryorchestration/internal/test" "testing" @@ -17,12 +18,18 @@ func TestRunMigrations(t *testing.T) { } ctx := context.Background() - _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{}) + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &serviceconfig.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) + + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + }) defer cleanup() - database.RunMigrations(ctx, &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../.."), - }) + err = migrations.Run(ctx, cfg) + assert.Nil(t, err) } func TestRunMigrationsNoDB(t *testing.T) { @@ -33,11 +40,12 @@ func TestRunMigrationsNoDB(t *testing.T) { t.Setenv("DB_HOST", "invalid_host") t.Setenv("DB_PORT", "5432") t.Setenv("DB_NAME", "invalid_name") - t.Setenv("DB_NOSSL", "1") + t.Setenv("DB_NOSSL", "true") + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &serviceconfig.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) - assert.Panics(t, func() { - database.RunMigrations(ctx, &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../.."), - }) - }) + err = migrations.Run(ctx, cfg) + assert.Error(t, err) } diff --git a/internal/database/migrations/migrationsprivate_test.go b/internal/database/migrations/migrationsprivate_test.go new file mode 100644 index 00000000..b12ec90f --- /dev/null +++ b/internal/database/migrations/migrationsprivate_test.go @@ -0,0 +1,18 @@ +package migrations + +import ( + "queryorchestration/internal/serviceconfig" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCreateDB(t *testing.T) { + t.Setenv("DB_HOST", "invalid_value") + cfg := &serviceconfig.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) + + err = createDB(cfg) + assert.Error(t, err) +} diff --git a/internal/database/migrationsprivate_test.go b/internal/database/migrationsprivate_test.go deleted file mode 100644 index 2875d043..00000000 --- a/internal/database/migrationsprivate_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package database - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestCreateDB(t *testing.T) { - t.Setenv("DB_HOST", "") - - assert.Panics(t, func() { createDB() }) -} diff --git a/internal/database/pool.go b/internal/database/pool.go deleted file mode 100644 index 5324c664..00000000 --- a/internal/database/pool.go +++ /dev/null @@ -1,17 +0,0 @@ -package database - -import ( - "context" - "queryorchestration/internal/database/repository" - - "github.com/jackc/pgx/v5" -) - -type Pool interface { - Begin(ctx context.Context) (pgx.Tx, error) -} - -type Connection struct { - Pool Pool - Queries *repository.Queries -} diff --git a/internal/database/repository/client_test.go b/internal/database/repository/client_test.go index 15c4f521..40dee16b 100644 --- a/internal/database/repository/client_test.go +++ b/internal/database/repository/client_test.go @@ -4,8 +4,8 @@ import ( "context" "os" "path" - "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" "queryorchestration/internal/test" "testing" @@ -14,13 +14,19 @@ import ( func TestClient(t *testing.T) { ctx := context.Background() - db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../../.."), - }}) + + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &serviceconfig.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) + + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) defer cleanup() - queries := repository.New(db.Pool) + queries := cfg.DBQueries id, err := queries.CreateClient(ctx, "example_client") assert.Nil(t, err) diff --git a/internal/database/repository/collector_test.go b/internal/database/repository/collector_test.go index 3606393b..c0f4d8bc 100644 --- a/internal/database/repository/collector_test.go +++ b/internal/database/repository/collector_test.go @@ -7,6 +7,7 @@ import ( "path" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" "queryorchestration/internal/test" "testing" @@ -16,13 +17,19 @@ import ( func TestCollector(t *testing.T) { ctx := context.Background() - db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../../.."), - }}) + + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &serviceconfig.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) + + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) defer cleanup() - queries := repository.New(db.Pool) + queries := cfg.DBQueries contextId, err := queries.CreateQuery(ctx, repository.QuerytypeContextFull) assert.Nil(t, err) diff --git a/internal/database/repository/document_test.go b/internal/database/repository/document_test.go index c8588ce3..169e4b35 100644 --- a/internal/database/repository/document_test.go +++ b/internal/database/repository/document_test.go @@ -4,8 +4,8 @@ import ( "context" "os" "path" - "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" "queryorchestration/internal/test" "testing" @@ -14,13 +14,19 @@ import ( func TestDocument(t *testing.T) { ctx := context.Background() - db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../../.."), - }}) + + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &serviceconfig.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) + + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) defer cleanup() - queries := repository.New(db.Pool) + queries := cfg.DBQueries clientId, err := queries.CreateClient(ctx, "example_client") assert.Nil(t, err) diff --git a/internal/database/repository/job_test.go b/internal/database/repository/job_test.go index f586320e..919611a2 100644 --- a/internal/database/repository/job_test.go +++ b/internal/database/repository/job_test.go @@ -4,8 +4,8 @@ import ( "context" "os" "path" - "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" "queryorchestration/internal/test" "testing" @@ -14,13 +14,18 @@ import ( func TestJob(t *testing.T) { ctx := context.Background() - db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../../.."), - }}) + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &serviceconfig.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) + + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) defer cleanup() - queries := repository.New(db.Pool) + queries := cfg.DBQueries clientId, err := queries.CreateClient(ctx, "example_client") assert.Nil(t, err) diff --git a/internal/database/repository/query_test.go b/internal/database/repository/query_test.go index 820de23e..106cbd89 100644 --- a/internal/database/repository/query_test.go +++ b/internal/database/repository/query_test.go @@ -6,6 +6,7 @@ import ( "path" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" "queryorchestration/internal/test" "testing" @@ -16,14 +17,18 @@ import ( func TestQueries(t *testing.T) { ctx := context.Background() + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &serviceconfig.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) - db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../../.."), - }}) + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) defer cleanup() - queries := repository.New(db.Pool) + queries := cfg.DBQueries contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) assert.Nil(t, err) @@ -162,13 +167,18 @@ func TestQueries(t *testing.T) { func TestQueryDependencyTree(t *testing.T) { ctx := context.Background() - db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../../.."), - }}) + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &serviceconfig.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) + + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) defer cleanup() - queries := repository.New(db.Pool) + queries := cfg.DBQueries contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) assert.Nil(t, err) @@ -230,13 +240,18 @@ func TestQueryDependencyTree(t *testing.T) { func TestQueriesList(t *testing.T) { ctx := context.Background() - db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: "../../..", - }}) + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &serviceconfig.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) + + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) defer cleanup() - queries := repository.New(db.Pool) + queries := cfg.DBQueries contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) assert.Nil(t, err) diff --git a/internal/database/repository/result_test.go b/internal/database/repository/result_test.go index 60efd68a..919b2de2 100644 --- a/internal/database/repository/result_test.go +++ b/internal/database/repository/result_test.go @@ -4,8 +4,8 @@ import ( "context" "os" "path" - "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" "queryorchestration/internal/test" "testing" @@ -16,13 +16,18 @@ import ( func TestResults(t *testing.T) { ctx := context.Background() - db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../../.."), - }}) + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &serviceconfig.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) + + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) defer cleanup() - queries := repository.New(db.Pool) + queries := cfg.DBQueries jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) assert.Nil(t, err) @@ -72,13 +77,18 @@ func TestResults(t *testing.T) { func TestResultValues(t *testing.T) { ctx := context.Background() - db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../../.."), - }}) + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &serviceconfig.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) + + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) defer cleanup() - queries := repository.New(db.Pool) + queries := cfg.DBQueries jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) assert.Nil(t, err) @@ -167,13 +177,18 @@ func TestResultValues(t *testing.T) { func TestUnsyncedQueries(t *testing.T) { ctx := context.Background() - db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../../.."), - }}) + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &serviceconfig.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) + + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) defer cleanup() - queries := repository.New(db.Pool) + queries := cfg.DBQueries clientId, err := queries.CreateClient(ctx, "example_client") assert.Nil(t, err) diff --git a/internal/document/create.go b/internal/document/create.go index 7380a800..3da75fdc 100644 --- a/internal/document/create.go +++ b/internal/document/create.go @@ -21,7 +21,7 @@ func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) { return uuid.Nil, err } - dbid, err := s.db.Queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + dbid, err := s.cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{ Jobid: database.MustToDBUUID(doc.JobID), Hash: hash, Location: doc.Location, diff --git a/internal/document/create_test.go b/internal/document/create_test.go index 32b27529..dc968643 100644 --- a/internal/document/create_test.go +++ b/internal/document/create_test.go @@ -5,6 +5,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/document" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -19,13 +20,11 @@ func TestCreate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := document.New(db) + svc := document.New(cfg) doc := document.Document{ ID: uuid.New(), diff --git a/internal/document/get.go b/internal/document/get.go index 7f2153b9..4e7d44fb 100644 --- a/internal/document/get.go +++ b/internal/document/get.go @@ -8,7 +8,7 @@ import ( ) func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Document, error) { - doc, err := s.db.Queries.GetDocument(ctx, database.MustToDBUUID(id)) + doc, err := s.cfg.GetDBQueries().GetDocument(ctx, database.MustToDBUUID(id)) if err != nil { return nil, err } diff --git a/internal/document/get_test.go b/internal/document/get_test.go index a4a2b569..3998b42a 100644 --- a/internal/document/get_test.go +++ b/internal/document/get_test.go @@ -5,6 +5,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/document" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -19,13 +20,11 @@ func TestGet(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := document.New(db) + svc := document.New(cfg) doc := document.Document{ ID: uuid.New(), diff --git a/internal/document/service.go b/internal/document/service.go index c8dc4462..b65870f7 100644 --- a/internal/document/service.go +++ b/internal/document/service.go @@ -1,7 +1,7 @@ package document import ( - "queryorchestration/internal/database" + "queryorchestration/internal/serviceconfig" "github.com/google/uuid" ) @@ -14,11 +14,11 @@ type Document struct { } type Service struct { - db *database.Connection + cfg serviceconfig.ConfigProvider } -func New(db *database.Connection) *Service { +func New(cfg serviceconfig.ConfigProvider) *Service { return &Service{ - db, + cfg, } } diff --git a/internal/export/service.go b/internal/export/service.go index d5ae6a25..2650da4e 100644 --- a/internal/export/service.go +++ b/internal/export/service.go @@ -1,13 +1,8 @@ package export -import "queryorchestration/internal/database" - type Service struct { - db *database.Connection } -func New(db *database.Connection) *Service { - return &Service{ - db, - } +func New() *Service { + return &Service{} } diff --git a/internal/export/service_test.go b/internal/export/service_test.go index 9ffcb523..7eef27d0 100644 --- a/internal/export/service_test.go +++ b/internal/export/service_test.go @@ -1,26 +1,13 @@ package export_test import ( - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" "queryorchestration/internal/export" "testing" - "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) func TestNew(t *testing.T) { - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - - svc := export.New(db) + svc := export.New() assert.NotNil(t, svc) } diff --git a/internal/job/collector/create.go b/internal/job/collector/create.go index bc3bba75..fc12d542 100644 --- a/internal/job/collector/create.go +++ b/internal/job/collector/create.go @@ -38,18 +38,6 @@ type dbCreateParams struct { Fields *map[string]pgtype.UUID } -type fields struct { - values *[]uuid.UUID -} - -func (f *fields) GetRequiredQueryIDs() *[]uuid.UUID { - return f.values -} - -func (f *fields) SetRequiredQueryIDs(v *[]uuid.UUID) { - f.values = v -} - func (s *Service) getCreateParams(ctx context.Context, params *CreateParams) (*dbCreateParams, error) { minClean := params.MinCleanVersion if minClean != nil { @@ -82,7 +70,7 @@ func (s *Service) getCreateParams(ctx context.Context, params *CreateParams) (*d func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) (uuid.UUID, error) { var dbID pgtype.UUID - err := database.ExecuteTransaction(ctx, s.db, func(ctx context.Context, qtx *repository.Queries) error { + err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error { dID, err := qtx.CreateCollector(ctx, params.JobID) if err != nil { return err diff --git a/internal/job/collector/create_test.go b/internal/job/collector/create_test.go index 2a06abfa..29026402 100644 --- a/internal/job/collector/create_test.go +++ b/internal/job/collector/create_test.go @@ -5,6 +5,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/job/collector" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -20,13 +21,11 @@ func TestCreate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := collector.New(db, &collector.Services{}) + svc := collector.New(cfg, &collector.Services{}) id := uuid.New() minCleanV := int32(2) diff --git a/internal/job/collector/createprivate_test.go b/internal/job/collector/createprivate_test.go index 2174ea75..e16ac8be 100644 --- a/internal/job/collector/createprivate_test.go +++ b/internal/job/collector/createprivate_test.go @@ -4,6 +4,7 @@ import ( "context" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -20,14 +21,12 @@ func TestGetCreateParams(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) svc := Service{ - db: db, + cfg: cfg, svc: &Services{}, } @@ -71,14 +70,12 @@ func TestSubmitCreate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) svc := Service{ - db: db, + cfg: cfg, svc: &Services{}, } @@ -109,13 +106,3 @@ func TestSubmitCreate(t *testing.T) { assert.Nil(t, err) assert.Equal(t, id, aid) } - -func TestFields(t *testing.T) { - f := fields{} - assert.Nil(t, f.GetRequiredQueryIDs()) - - values := []uuid.UUID{uuid.New()} - f.SetRequiredQueryIDs(&values) - assert.NotNil(t, f.values) - assert.ElementsMatch(t, *f.values, *f.GetRequiredQueryIDs()) -} diff --git a/internal/job/collector/get.go b/internal/job/collector/get.go index 90948b4f..23424ee5 100644 --- a/internal/job/collector/get.go +++ b/internal/job/collector/get.go @@ -9,7 +9,7 @@ import ( ) func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Collector, error) { - dbColl, err := s.db.Queries.GetCollector(ctx, database.MustToDBUUID(id)) + dbColl, err := s.cfg.GetDBQueries().GetCollector(ctx, database.MustToDBUUID(id)) if err != nil { return nil, err } @@ -18,7 +18,7 @@ func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Collector, error) { } func (s *Service) GetByJobID(ctx context.Context, jobID uuid.UUID) (*Collector, error) { - dbColl, err := s.db.Queries.GetCollectorByJobID(ctx, database.MustToDBUUID(jobID)) + dbColl, err := s.cfg.GetDBQueries().GetCollectorByJobID(ctx, database.MustToDBUUID(jobID)) if err != nil { return nil, err } @@ -27,7 +27,7 @@ func (s *Service) GetByJobID(ctx context.Context, jobID uuid.UUID) (*Collector, } func (s *Service) ListQueries(ctx context.Context, id uuid.UUID) ([]*resultprocessor.Query, error) { - queries, err := s.db.Queries.ListCollectorQueries(ctx, database.MustToDBUUID(id)) + queries, err := s.cfg.GetDBQueries().ListCollectorQueries(ctx, database.MustToDBUUID(id)) if err != nil { return nil, err } diff --git a/internal/job/collector/get_test.go b/internal/job/collector/get_test.go index ef303844..2434ae47 100644 --- a/internal/job/collector/get_test.go +++ b/internal/job/collector/get_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/job/collector" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -20,13 +21,11 @@ func TestGet(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := collector.New(db, &collector.Services{}) + svc := collector.New(cfg, &collector.Services{}) minCleanV := int32(2) minTextV := int32(4) @@ -58,13 +57,11 @@ func TestGetByJobID(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := collector.New(db, &collector.Services{}) + svc := collector.New(cfg, &collector.Services{}) minCleanV := int32(2) minTextV := int32(4) diff --git a/internal/job/collector/service.go b/internal/job/collector/service.go index a50d6a66..6024b356 100644 --- a/internal/job/collector/service.go +++ b/internal/job/collector/service.go @@ -1,9 +1,9 @@ package collector import ( - "queryorchestration/internal/database" documentclean "queryorchestration/internal/document/clean" documenttext "queryorchestration/internal/document/text" + "queryorchestration/internal/serviceconfig" "github.com/google/uuid" ) @@ -24,13 +24,13 @@ type Services struct { } type Service struct { - db *database.Connection + cfg serviceconfig.ConfigProvider svc *Services } -func New(db *database.Connection, svc *Services) *Service { +func New(cfg serviceconfig.ConfigProvider, svc *Services) *Service { return &Service{ - db, + cfg, svc, } } diff --git a/internal/job/collector/service_test.go b/internal/job/collector/service_test.go index dea647dd..f442ca47 100644 --- a/internal/job/collector/service_test.go +++ b/internal/job/collector/service_test.go @@ -1,9 +1,9 @@ package collector_test import ( - "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/job/collector" + "queryorchestration/internal/serviceconfig" "testing" "github.com/pashagolub/pgxmock/v3" @@ -15,12 +15,10 @@ func TestService(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := collector.New(db, &collector.Services{}) + svc := collector.New(cfg, &collector.Services{}) assert.NotNil(t, svc) } diff --git a/internal/job/collector/update.go b/internal/job/collector/update.go index 085c5600..cf8e6468 100644 --- a/internal/job/collector/update.go +++ b/internal/job/collector/update.go @@ -155,7 +155,7 @@ func (s *Service) normalizeFieldsToDB(ctx context.Context, ofields *map[string]u return nil, errors.New("duplicate output fields") } - exist, err := s.db.Queries.AllQueriesExist(ctx, dbids) + exist, err := s.cfg.GetDBQueries().AllQueriesExist(ctx, dbids) if err != nil { return nil, err } else if !exist { @@ -166,7 +166,7 @@ func (s *Service) normalizeFieldsToDB(ctx context.Context, ofields *map[string]u } func (s *Service) submitUpdate(ctx context.Context, current *Collector, params *dbUpdateParams) error { - err := database.ExecuteTransaction(ctx, s.db, func(ctx context.Context, qtx *repository.Queries) error { + err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error { latestVersion := current.LatestVersion + 1 id := database.MustToDBUUID(current.ID) diff --git a/internal/job/collector/update_test.go b/internal/job/collector/update_test.go index 3caa0c21..b010f204 100644 --- a/internal/job/collector/update_test.go +++ b/internal/job/collector/update_test.go @@ -7,6 +7,7 @@ import ( documentclean "queryorchestration/internal/document/clean" documenttext "queryorchestration/internal/document/text" "queryorchestration/internal/job/collector" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -22,13 +23,11 @@ func TestUpdate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := collector.New(db, &collector.Services{ + svc := collector.New(cfg, &collector.Services{ Clean: documentclean.New(), Text: documenttext.New(), }) diff --git a/internal/job/collector/updateprivate_test.go b/internal/job/collector/updateprivate_test.go index d0fa87be..c9717ee1 100644 --- a/internal/job/collector/updateprivate_test.go +++ b/internal/job/collector/updateprivate_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/database/repository" documentclean "queryorchestration/internal/document/clean" documenttext "queryorchestration/internal/document/text" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -22,14 +23,12 @@ func TestGetUpdateParams(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) svc := Service{ - db: db, + cfg: cfg, svc: &Services{}, } @@ -86,14 +85,12 @@ func TestSubmitUpdate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) svc := Service{ - db: db, + cfg: cfg, svc: &Services{}, } @@ -155,14 +152,12 @@ func TestNormalizeFieldsToDB(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) svc := Service{ - db: db, + cfg: cfg, svc: &Services{}, } diff --git a/internal/job/create.go b/internal/job/create.go index eb5cd05a..97d1b56b 100644 --- a/internal/job/create.go +++ b/internal/job/create.go @@ -9,7 +9,7 @@ import ( ) func (s *Service) Create(ctx context.Context, clientID uuid.UUID) (uuid.UUID, error) { - did, err := s.db.Queries.CreateJob(ctx, database.MustToDBUUID(clientID)) + did, err := s.cfg.GetDBQueries().CreateJob(ctx, database.MustToDBUUID(clientID)) if err != nil { return uuid.Nil, err } diff --git a/internal/job/create_test.go b/internal/job/create_test.go index 1c1035be..0ee05bc0 100644 --- a/internal/job/create_test.go +++ b/internal/job/create_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/database/repository" "queryorchestration/internal/job" "queryorchestration/internal/job/collector" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -21,14 +22,12 @@ func TestCreate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := job.New(db, &job.Services{ - Collector: collector.New(db, &collector.Services{}), + svc := job.New(cfg, &job.Services{ + Collector: collector.New(cfg, &collector.Services{}), }) job := job.Job{ diff --git a/internal/job/get.go b/internal/job/get.go index 5ebebe9a..51b28311 100644 --- a/internal/job/get.go +++ b/internal/job/get.go @@ -8,7 +8,7 @@ import ( ) func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Job, error) { - job, err := s.db.Queries.GetJob(ctx, database.MustToDBUUID(id)) + job, err := s.cfg.GetDBQueries().GetJob(ctx, database.MustToDBUUID(id)) if err != nil { return nil, err } diff --git a/internal/job/get_test.go b/internal/job/get_test.go index 3f9bcf91..57d8f849 100644 --- a/internal/job/get_test.go +++ b/internal/job/get_test.go @@ -7,6 +7,7 @@ import ( "queryorchestration/internal/database/repository" "queryorchestration/internal/job" "queryorchestration/internal/job/collector" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -21,15 +22,13 @@ func TestGet(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := job.New(db, &job.Services{ - Collector: collector.New(db, &collector.Services{}), - Client: client.New(db), + svc := job.New(cfg, &job.Services{ + Collector: collector.New(cfg, &collector.Services{}), + Client: client.New(cfg), }) j := job.Job{ diff --git a/internal/job/service.go b/internal/job/service.go index ae2bb2e8..964be363 100644 --- a/internal/job/service.go +++ b/internal/job/service.go @@ -2,8 +2,8 @@ package job import ( "queryorchestration/internal/client" - "queryorchestration/internal/database" "queryorchestration/internal/job/collector" + "queryorchestration/internal/serviceconfig" "github.com/google/uuid" ) @@ -20,13 +20,13 @@ type Services struct { } type Service struct { - db *database.Connection + cfg serviceconfig.ConfigProvider svc *Services } -func New(db *database.Connection, svc *Services) *Service { +func New(cfg serviceconfig.ConfigProvider, svc *Services) *Service { return &Service{ - db, + cfg, svc, } } diff --git a/internal/job/service_test.go b/internal/job/service_test.go index eff737fa..8715148d 100644 --- a/internal/job/service_test.go +++ b/internal/job/service_test.go @@ -1,9 +1,9 @@ package job_test import ( - "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/job" + "queryorchestration/internal/serviceconfig" "testing" "github.com/pashagolub/pgxmock/v3" @@ -15,12 +15,10 @@ func TestService(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := job.New(db, &job.Services{}) + svc := job.New(cfg, &job.Services{}) assert.NotNil(t, svc) } diff --git a/internal/job/update.go b/internal/job/update.go index 3c19893e..3a89ee61 100644 --- a/internal/job/update.go +++ b/internal/job/update.go @@ -51,7 +51,7 @@ func (s *Service) normalizeUpdate(ctx context.Context, current *Job, update *Upd } func (s *Service) submitUpdate(ctx context.Context, update *Update) error { - err := database.ExecuteTransaction(ctx, s.db, func(ctx context.Context, q *repository.Queries) error { + err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error { if update.CanSync != nil { err := q.AddJobCanSync(ctx, &repository.AddJobCanSyncParams{ Jobid: database.MustToDBUUID(update.ID), diff --git a/internal/job/update_test.go b/internal/job/update_test.go index 2d715517..bd909780 100644 --- a/internal/job/update_test.go +++ b/internal/job/update_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/job" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -20,14 +21,12 @@ func TestUpdate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := job.New(db, &job.Services{ - Client: client.New(db), + svc := job.New(cfg, &job.Services{ + Client: client.New(cfg), }) j := job.Job{ diff --git a/internal/job/updateprivate_test.go b/internal/job/updateprivate_test.go index 3b1a592d..16ebe93c 100644 --- a/internal/job/updateprivate_test.go +++ b/internal/job/updateprivate_test.go @@ -5,6 +5,7 @@ import ( "queryorchestration/internal/client" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -19,14 +20,12 @@ func TestNormalizeCanSync(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := New(db, &Services{ - Client: client.New(db), + svc := New(cfg, &Services{ + Client: client.New(cfg), }) j := Job{ @@ -99,14 +98,12 @@ func TestNormalizeUpdate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := New(db, &Services{ - Client: client.New(db), + svc := New(cfg, &Services{ + Client: client.New(cfg), }) j := Job{ @@ -134,14 +131,12 @@ func TestSubmitUpdate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) svc := Service{ - db: db, + cfg: cfg, } j := Job{ diff --git a/internal/query/create.go b/internal/query/create.go index 91a06cfb..e2b4dab1 100644 --- a/internal/query/create.go +++ b/internal/query/create.go @@ -58,7 +58,7 @@ func (s *Service) submitCreate(ctx context.Context, entity *resultprocessor.Crea } var dbID pgtype.UUID - err = database.ExecuteTransaction(ctx, s.db, func(ctx context.Context, qtx *repository.Queries) error { + err = s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error { dbID, err = qtx.CreateQuery(ctx, query.Type) if err != nil { return err @@ -106,9 +106,9 @@ func (s *Service) submitCreate(ctx context.Context, entity *resultprocessor.Crea func (s *Service) getCreator(qType resultprocessor.Type) (resultprocessor.Creator, error) { switch qType { case resultprocessor.TypeJsonExtractor: - return jsonextractor.NewCreator(s.db), nil + return jsonextractor.NewCreator(), nil case resultprocessor.TypeContextFull: - return contextfull.NewCreator(s.db), nil + return contextfull.NewCreator(), nil default: return nil, fmt.Errorf("attempting to process invalid query type") } diff --git a/internal/query/create_test.go b/internal/query/create_test.go index 304d3b5e..58c8ade3 100644 --- a/internal/query/create_test.go +++ b/internal/query/create_test.go @@ -7,6 +7,7 @@ import ( "queryorchestration/internal/database/repository" "queryorchestration/internal/query" resultprocessor "queryorchestration/internal/query/result/processor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -22,12 +23,10 @@ func TestCreate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := query.New(db, &query.Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := query.New(cfg, &query.Services{}) config := "{\"path\":\"example_path\"}" q := query.Query{ @@ -76,12 +75,10 @@ func TestCreateMinimal(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := query.New(db, &query.Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := query.New(cfg, &query.Services{}) q := query.Query{ ID: uuid.New(), @@ -113,12 +110,10 @@ func TestCreateRollback(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := query.New(db, &query.Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := query.New(cfg, &query.Services{}) create := &resultprocessor.Create{ Type: resultprocessor.TypeJsonExtractor, diff --git a/internal/query/createprivate_test.go b/internal/query/createprivate_test.go index ce9649bd..7e6f216b 100644 --- a/internal/query/createprivate_test.go +++ b/internal/query/createprivate_test.go @@ -5,6 +5,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -18,12 +19,10 @@ func TestGetCreator(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := New(db, &Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg, &Services{}) queryType := resultprocessor.Type(resultprocessor.TypeContextFull) creator, err := svc.getCreator(queryType) @@ -82,12 +81,10 @@ func TestSubmitCreate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := New(db, &Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg, &Services{}) config := "{\"path\":\"example_path\"}" q := Query{ @@ -132,12 +129,10 @@ func TestSubmitCreateNoReqsOrConfig(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := New(db, &Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg, &Services{}) q := Query{ ID: uuid.New(), @@ -169,12 +164,10 @@ func TestNormalizeCreate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := New(db, &Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg, &Services{}) create := &resultprocessor.Create{ Type: resultprocessor.TypeJsonExtractor, diff --git a/internal/query/get.go b/internal/query/get.go index 4e25e01e..5383e899 100644 --- a/internal/query/get.go +++ b/internal/query/get.go @@ -19,7 +19,7 @@ type Query struct { } func (s *Service) GetWithVersion(ctx context.Context, id uuid.UUID, version int32) (*Query, error) { - query, err := s.db.Queries.GetQueryWithVersion(ctx, &repository.GetQueryWithVersionParams{ + query, err := s.cfg.GetDBQueries().GetQueryWithVersion(ctx, &repository.GetQueryWithVersionParams{ ID: database.MustToDBUUID(id), Addedversion: version, }) @@ -31,7 +31,7 @@ func (s *Service) GetWithVersion(ctx context.Context, id uuid.UUID, version int3 } func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Query, error) { - query, err := s.db.Queries.GetQuery(ctx, database.MustToDBUUID(id)) + query, err := s.cfg.GetDBQueries().GetQuery(ctx, database.MustToDBUUID(id)) if err != nil { return nil, err } diff --git a/internal/query/get_test.go b/internal/query/get_test.go index b1f16ecf..36980b02 100644 --- a/internal/query/get_test.go +++ b/internal/query/get_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/database/repository" "queryorchestration/internal/query" resultprocessor "queryorchestration/internal/query/result/processor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -20,12 +21,10 @@ func TestGet(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := query.New(db, &query.Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := query.New(cfg, &query.Services{}) config := "{\"path\":\"example_path\"}" query := query.Query{ @@ -59,12 +58,10 @@ func TestGetWithVersion(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := query.New(db, &query.Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := query.New(cfg, &query.Services{}) config := "{\"path\":\"example_path\"}" query := query.Query{ diff --git a/internal/query/list.go b/internal/query/list.go index 5cf4d5f4..ddfb9f5e 100644 --- a/internal/query/list.go +++ b/internal/query/list.go @@ -8,7 +8,7 @@ import ( ) func (s *Service) List(ctx context.Context) ([]*Query, error) { - dbQueries, err := s.db.Queries.ListQueries(ctx) + dbQueries, err := s.cfg.GetDBQueries().ListQueries(ctx) if err != nil { return nil, err } @@ -22,7 +22,7 @@ func (s *Service) List(ctx context.Context) ([]*Query, error) { } func (s *Service) ListById(ctx context.Context, ids []uuid.UUID) ([]*Query, error) { - dbQueries, err := s.db.Queries.ListQueriesById(ctx, database.MustToDBUUIDArray(ids)) + dbQueries, err := s.cfg.GetDBQueries().ListQueriesById(ctx, database.MustToDBUUIDArray(ids)) if err != nil { return nil, err } diff --git a/internal/query/list_test.go b/internal/query/list_test.go index 7a91eb47..21fa9f05 100644 --- a/internal/query/list_test.go +++ b/internal/query/list_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/database/repository" "queryorchestration/internal/query" resultprocessor "queryorchestration/internal/query/result/processor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -21,12 +22,10 @@ func TestList(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := query.New(db, &query.Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := query.New(cfg, &query.Services{}) config := "{\"path\":\"example_path\"}" q := &query.Query{ @@ -60,12 +59,10 @@ func TestListById(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := query.New(db, &query.Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := query.New(cfg, &query.Services{}) config := "{\"path\":\"example_path\"}" q := &query.Query{ diff --git a/internal/query/normalize.go b/internal/query/normalize.go index 10d0cfbf..2728fd17 100644 --- a/internal/query/normalize.go +++ b/internal/query/normalize.go @@ -73,7 +73,7 @@ func (s *Service) NormalizeQueryIDs(ctx context.Context, ids RequiredQueryIDs) e dbids := database.MustToDBUUIDArray(dedup) - exist, err := s.db.Queries.AllQueriesExist(ctx, dbids) + exist, err := s.cfg.GetDBQueries().AllQueriesExist(ctx, dbids) if err != nil { return err } else if !exist { diff --git a/internal/query/normalize_test.go b/internal/query/normalize_test.go index 8e5d444c..002d0a73 100644 --- a/internal/query/normalize_test.go +++ b/internal/query/normalize_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -74,12 +75,10 @@ func TestNormalizeQueryIDs(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - s := Service{db: db} + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + s := Service{cfg: cfg} err = s.NormalizeQueryIDs(ctx, nil) assert.Nil(t, err) diff --git a/internal/query/result/get.go b/internal/query/result/get.go index 07831b45..0585dcb1 100644 --- a/internal/query/result/get.go +++ b/internal/query/result/get.go @@ -22,7 +22,7 @@ type GetValueWithVersionParams struct { } func (s *Service) GetValueWithVersion(ctx context.Context, params *GetValueWithVersionParams) (resultprocessor.Value, error) { - res, err := s.db.Queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{ + res, err := s.cfg.GetDBQueries().GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{ Queryid: database.MustToDBUUID(params.QueryID), Queryversion: params.QueryVersion, Documentid: database.MustToDBUUID(params.DocumentID), diff --git a/internal/query/result/get_test.go b/internal/query/result/get_test.go index d6fb8b51..980528dc 100644 --- a/internal/query/result/get_test.go +++ b/internal/query/result/get_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -34,12 +35,11 @@ func TestGetValueWithVersion(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := New(db) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := New(cfg) params := &GetValueWithVersionParams{ Type: resultprocessor.TypeJsonExtractor, diff --git a/internal/query/result/list.go b/internal/query/result/list.go index 1908dada..064acec9 100644 --- a/internal/query/result/list.go +++ b/internal/query/result/list.go @@ -18,7 +18,7 @@ type ListQueryRequirementValuesParams struct { } func (s *Service) ListQueryRequirementValues(ctx context.Context, params *ListQueryRequirementValuesParams) (*[]resultprocessor.Value, error) { - qResults, err := s.db.Queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ + qResults, err := s.cfg.GetDBQueries().ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ Queryid: database.MustToDBUUID(params.QueryID), Documentid: database.MustToDBUUID(params.DocumentID), Addedversion: params.QueryVersion, diff --git a/internal/query/result/list_test.go b/internal/query/result/list_test.go index 705cc3ce..964efe51 100644 --- a/internal/query/result/list_test.go +++ b/internal/query/result/list_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -21,13 +22,11 @@ func TestListQueryRequirementValues(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) svc := Service{ - db: db, + cfg: cfg, } params := &ListQueryRequirementValuesParams{ diff --git a/internal/query/result/process.go b/internal/query/result/process.go index 49438a4a..ba931a62 100644 --- a/internal/query/result/process.go +++ b/internal/query/result/process.go @@ -61,7 +61,7 @@ func (s *Service) listRequiredValues(ctx context.Context, p *Process) (*[]result func (s *Service) getProcessor(queryType resultprocessor.Type) (resultprocessor.Processor, error) { switch queryType { case resultprocessor.TypeJsonExtractor: - return jsonextractor.NewExtractor(s.db), nil + return jsonextractor.NewExtractor(s.cfg), nil case resultprocessor.TypeContextFull: return contextfull.NewExtractor(), nil default: diff --git a/internal/query/result/process_test.go b/internal/query/result/process_test.go index e749326d..ad9fb5ae 100644 --- a/internal/query/result/process_test.go +++ b/internal/query/result/process_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -21,17 +22,15 @@ func TestProcess(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) svc := Service{ - db: db, + cfg: cfg, } - cfg := "{\"path\":\"examplekey\"}" + qcfg := "{\"path\":\"examplekey\"}" params := Process{ DocumentID: uuid.New(), MinCleanVersion: 1, @@ -39,7 +38,7 @@ func TestProcess(t *testing.T) { Query: &resultprocessor.Query{ ID: uuid.New(), Version: 2, - Config: &cfg, + Config: &qcfg, RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, @@ -53,7 +52,7 @@ func TestProcess(t *testing.T) { ) pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version).WillReturnRows( pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(cfg)), + AddRow(pgtype.UUID{}, []byte(qcfg)), ) val, err := svc.Process(ctx, ¶ms) @@ -69,14 +68,12 @@ func TestListRequiredValue(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) svc := Service{ - db: db, + cfg: cfg, } pr, err := svc.listRequiredValues(ctx, nil) diff --git a/internal/query/result/service.go b/internal/query/result/service.go index 9810a15f..575da710 100644 --- a/internal/query/result/service.go +++ b/internal/query/result/service.go @@ -1,7 +1,7 @@ package result import ( - "queryorchestration/internal/database" + "queryorchestration/internal/serviceconfig" "github.com/google/uuid" ) @@ -13,11 +13,11 @@ type Result struct { } type Service struct { - db *database.Connection + cfg serviceconfig.ConfigProvider } -func New(db *database.Connection) *Service { +func New(cfg serviceconfig.ConfigProvider) *Service { return &Service{ - db, + cfg, } } diff --git a/internal/query/result/service_test.go b/internal/query/result/service_test.go index 65093aeb..1189f1d4 100644 --- a/internal/query/result/service_test.go +++ b/internal/query/result/service_test.go @@ -1,9 +1,9 @@ package result_test import ( - "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/query/result" + "queryorchestration/internal/serviceconfig" "testing" "github.com/pashagolub/pgxmock/v3" @@ -15,11 +15,9 @@ func TestService(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := result.New(db) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := result.New(cfg) assert.NotNil(t, svc) } diff --git a/internal/query/result/set.go b/internal/query/result/set.go index d45e167d..34c91927 100644 --- a/internal/query/result/set.go +++ b/internal/query/result/set.go @@ -27,7 +27,7 @@ func (s *Service) Set(ctx context.Context, params *Set) (uuid.UUID, error) { return uuid.Nil, err } - dbId, err := s.db.Queries.SetResult(ctx, &repository.SetResultParams{ + dbId, err := s.cfg.GetDBQueries().SetResult(ctx, &repository.SetResultParams{ Queryid: database.MustToDBUUID(params.Query.ID), Documentid: database.MustToDBUUID(params.DocumentID), Value: value.GetStoreValue(), diff --git a/internal/query/result/set_test.go b/internal/query/result/set_test.go index c2e94b3a..e278af50 100644 --- a/internal/query/result/set_test.go +++ b/internal/query/result/set_test.go @@ -5,6 +5,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -20,18 +21,16 @@ func TestSet(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) svc := Service{ - db: db, + cfg: cfg, } rid := uuid.New() - cfg := "{\"path\":\"examplekey\"}" + qcfg := "{\"path\":\"examplekey\"}" params := Set{ DocumentID: uuid.New(), CleanVersion: 1, @@ -39,7 +38,7 @@ func TestSet(t *testing.T) { Query: &resultprocessor.Query{ ID: uuid.New(), Version: 2, - Config: &cfg, + Config: &qcfg, RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, @@ -53,7 +52,7 @@ func TestSet(t *testing.T) { ) pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version).WillReturnRows( pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(cfg)), + AddRow(pgtype.UUID{}, []byte(qcfg)), ) pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(params.Query.ID), database.MustToDBUUID(params.DocumentID), pgxmock.AnyArg(), params.CleanVersion, params.TextVersion, params.Query.Version). WillReturnRows( diff --git a/internal/query/result/sync.go b/internal/query/result/sync.go index 84e36a8a..1c1567b0 100644 --- a/internal/query/result/sync.go +++ b/internal/query/result/sync.go @@ -9,7 +9,7 @@ import ( ) func (s *Service) ListUnsyncedQueriesByDocId(ctx context.Context, id uuid.UUID) ([]*resultprocessor.Query, error) { - qs, err := s.db.Queries.ListUnsyncedQueriesByDocId(ctx, database.MustToDBUUID(id)) + qs, err := s.cfg.GetDBQueries().ListUnsyncedQueriesByDocId(ctx, database.MustToDBUUID(id)) if err != nil { return nil, err } diff --git a/internal/query/result/sync_test.go b/internal/query/result/sync_test.go index 6f963b04..204c29f8 100644 --- a/internal/query/result/sync_test.go +++ b/internal/query/result/sync_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/database/repository" "queryorchestration/internal/query/result" resultprocessor "queryorchestration/internal/query/result/processor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -21,12 +22,10 @@ func TestListUnsyncedQueriesByDocId(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := result.New(db) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := result.New(cfg) documentId := uuid.New() actualQs := []*resultprocessor.Query{ diff --git a/internal/query/service.go b/internal/query/service.go index f9aad383..1622c811 100644 --- a/internal/query/service.go +++ b/internal/query/service.go @@ -1,11 +1,11 @@ package query import ( - "queryorchestration/internal/database" "queryorchestration/internal/document" documenttext "queryorchestration/internal/document/text" "queryorchestration/internal/job/collector" "queryorchestration/internal/query/result" + "queryorchestration/internal/serviceconfig" "github.com/google/uuid" ) @@ -18,13 +18,13 @@ type Services struct { } type Service struct { - db *database.Connection + cfg serviceconfig.ConfigProvider svc *Services } -func New(db *database.Connection, svc *Services) *Service { +func New(cfg serviceconfig.ConfigProvider, svc *Services) *Service { return &Service{ - db, + cfg, svc, } } diff --git a/internal/query/service_test.go b/internal/query/service_test.go index eebf8452..930b33b2 100644 --- a/internal/query/service_test.go +++ b/internal/query/service_test.go @@ -1,9 +1,9 @@ package query_test import ( - "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/query" + "queryorchestration/internal/serviceconfig" "testing" "github.com/pashagolub/pgxmock/v3" @@ -15,11 +15,9 @@ func TestService(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := query.New(db, &query.Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := query.New(cfg, &query.Services{}) assert.NotNil(t, svc) } diff --git a/internal/query/sync_test.go b/internal/query/sync_test.go index b9d5b3f9..83b5fc0a 100644 --- a/internal/query/sync_test.go +++ b/internal/query/sync_test.go @@ -7,6 +7,7 @@ import ( "queryorchestration/internal/job/collector" "queryorchestration/internal/query/result" resultprocessor "queryorchestration/internal/query/result/processor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -186,14 +187,12 @@ func TestProcessBatch(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) svc := Service{ svc: &Services{ - Result: result.New(db), + Result: result.New(cfg), }, } @@ -246,14 +245,12 @@ func TestSync(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) svc := Service{ svc: &Services{ - Result: result.New(db), + Result: result.New(cfg), }, } diff --git a/internal/query/test_test.go b/internal/query/test_test.go index b08809e2..8748b3e8 100644 --- a/internal/query/test_test.go +++ b/internal/query/test_test.go @@ -10,6 +10,7 @@ import ( "queryorchestration/internal/job/collector" "queryorchestration/internal/query" "queryorchestration/internal/query/result" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -25,23 +26,21 @@ func TestTest(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) text := documenttext.New() clean := documentclean.New() - col := collector.New(db, &collector.Services{ + col := collector.New(cfg, &collector.Services{ Text: text, Clean: clean, }) - docsvc := document.New(db) - svc := query.New(db, &query.Services{ + docsvc := document.New(cfg) + svc := query.New(cfg, &query.Services{ Text: text, Document: docsvc, Collector: col, - Result: result.New(db), + Result: result.New(cfg), }) coll := collector.Collector{ diff --git a/internal/query/types/contextFull/creator.go b/internal/query/types/contextFull/creator.go index a0311669..0bebe4da 100644 --- a/internal/query/types/contextFull/creator.go +++ b/internal/query/types/contextFull/creator.go @@ -2,19 +2,17 @@ package contextfull import ( "context" - "queryorchestration/internal/database" resultprocessor "queryorchestration/internal/query/result/processor" ) type Creator struct { - db *database.Connection } -func NewCreator(db *database.Connection) Creator { - return Creator{db} +func NewCreator() *Creator { + return &Creator{} } -func (s Creator) Validate(ctx context.Context, entity *resultprocessor.Create) error { +func (s *Creator) Validate(ctx context.Context, entity *resultprocessor.Create) error { // TODO // Type, RequiredQueryIDs, Config return nil diff --git a/internal/query/types/contextFull/creator_test.go b/internal/query/types/contextFull/creator_test.go index 3f14de3f..da4fbeb6 100644 --- a/internal/query/types/contextFull/creator_test.go +++ b/internal/query/types/contextFull/creator_test.go @@ -2,10 +2,10 @@ package contextfull_test import ( "context" - "queryorchestration/internal/database" "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" contextfull "queryorchestration/internal/query/types/contextFull" + "queryorchestration/internal/serviceconfig" "testing" "github.com/pashagolub/pgxmock/v3" @@ -19,13 +19,11 @@ func TestCreatorValidate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := contextfull.NewCreator(db) + svc := contextfull.NewCreator() assert.NotNil(t, svc) entity := &resultprocessor.Create{ diff --git a/internal/query/types/contextFull/result.go b/internal/query/types/contextFull/result.go index 32abfb22..97809efc 100644 --- a/internal/query/types/contextFull/result.go +++ b/internal/query/types/contextFull/result.go @@ -8,11 +8,11 @@ type Result struct { value string } -func NewResult(value string) Result { - return Result{value} +func NewResult(value string) *Result { + return &Result{value} } -func (r Result) GetValue(ctx context.Context) (string, error) { +func (r *Result) GetValue(ctx context.Context) (string, error) { // TODO - get value from s3 return r.value, nil } diff --git a/internal/query/types/contextFull/service.go b/internal/query/types/contextFull/service.go index 2649610e..3950f444 100644 --- a/internal/query/types/contextFull/service.go +++ b/internal/query/types/contextFull/service.go @@ -9,11 +9,11 @@ import ( type Extractor struct { } -func NewExtractor() Extractor { - return Extractor{} +func NewExtractor() *Extractor { + return &Extractor{} } -func (e Extractor) Process(ctx context.Context, query *resultprocessor.Query, values *[]resultprocessor.Value) (string, error) { +func (e *Extractor) Process(ctx context.Context, query *resultprocessor.Query, values *[]resultprocessor.Value) (string, error) { if values != nil && len(*values) > 0 { return "", errors.New("no requirements expected") } diff --git a/internal/query/types/contextFull/updator.go b/internal/query/types/contextFull/updator.go index ddb1749e..935652eb 100644 --- a/internal/query/types/contextFull/updator.go +++ b/internal/query/types/contextFull/updator.go @@ -2,19 +2,16 @@ package contextfull import ( "context" - "queryorchestration/internal/database" resultprocessor "queryorchestration/internal/query/result/processor" ) -type Updator struct { - db *database.Connection +type Updator struct{} + +func NewUpdator() *Updator { + return &Updator{} } -func NewUpdator(db *database.Connection) Updator { - return Updator{db} -} - -func (s Updator) Validate(ctx context.Context, current *resultprocessor.Query, entity *resultprocessor.Update) error { +func (s *Updator) Validate(ctx context.Context, current *resultprocessor.Query, entity *resultprocessor.Update) error { // TODO // Type, RequiredQueryIDs, Config return nil diff --git a/internal/query/types/contextFull/updator_test.go b/internal/query/types/contextFull/updator_test.go index d7e5fedc..bd5f09a4 100644 --- a/internal/query/types/contextFull/updator_test.go +++ b/internal/query/types/contextFull/updator_test.go @@ -2,10 +2,10 @@ package contextfull_test import ( "context" - "queryorchestration/internal/database" "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" contextfull "queryorchestration/internal/query/types/contextFull" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -20,13 +20,11 @@ func TestUpdatorValidate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := contextfull.NewUpdator(db) + svc := contextfull.NewUpdator() assert.NotNil(t, svc) current := &resultprocessor.Query{ diff --git a/internal/query/types/jsonExtractor/creator.go b/internal/query/types/jsonExtractor/creator.go index fa147462..a71db13a 100644 --- a/internal/query/types/jsonExtractor/creator.go +++ b/internal/query/types/jsonExtractor/creator.go @@ -2,19 +2,16 @@ package jsonextractor import ( "context" - "queryorchestration/internal/database" resultprocessor "queryorchestration/internal/query/result/processor" ) -type Creator struct { - db *database.Connection +type Creator struct{} + +func NewCreator() *Creator { + return &Creator{} } -func NewCreator(db *database.Connection) Creator { - return Creator{db} -} - -func (s Creator) Validate(ctx context.Context, entity *resultprocessor.Create) error { +func (s *Creator) Validate(ctx context.Context, entity *resultprocessor.Create) error { // TODO return nil } diff --git a/internal/query/types/jsonExtractor/creator_test.go b/internal/query/types/jsonExtractor/creator_test.go index 0bde4356..5bc55f0a 100644 --- a/internal/query/types/jsonExtractor/creator_test.go +++ b/internal/query/types/jsonExtractor/creator_test.go @@ -2,10 +2,10 @@ package jsonextractor_test import ( "context" - "queryorchestration/internal/database" "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/pashagolub/pgxmock/v3" @@ -19,13 +19,11 @@ func TestCreatorValidate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := jsonextractor.NewCreator(db) + svc := jsonextractor.NewCreator() assert.NotNil(t, svc) entity := &resultprocessor.Create{ diff --git a/internal/query/types/jsonExtractor/process_test.go b/internal/query/types/jsonExtractor/process_test.go index b72875aa..7dbb0ebd 100644 --- a/internal/query/types/jsonExtractor/process_test.go +++ b/internal/query/types/jsonExtractor/process_test.go @@ -8,6 +8,7 @@ import ( resultprocessor "queryorchestration/internal/query/result/processor" contextfull "queryorchestration/internal/query/types/contextFull" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -23,13 +24,11 @@ func TestJSONProcess(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - extractor := jsonextractor.NewExtractor(db) + extractor := jsonextractor.NewExtractor(cfg) query := &resultprocessor.Query{ ID: uuid.New(), @@ -93,13 +92,11 @@ func TestJSONProcessJSON(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - extractor := jsonextractor.NewExtractor(db) + extractor := jsonextractor.NewExtractor(cfg) query := &resultprocessor.Query{ ID: uuid.New(), @@ -178,13 +175,11 @@ func TestJSONProcessResults(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - extractor := jsonextractor.NewExtractor(db) + extractor := jsonextractor.NewExtractor(cfg) query := &resultprocessor.Query{ ID: uuid.New(), diff --git a/internal/query/types/jsonExtractor/result.go b/internal/query/types/jsonExtractor/result.go index 682763d3..08b8a510 100644 --- a/internal/query/types/jsonExtractor/result.go +++ b/internal/query/types/jsonExtractor/result.go @@ -8,11 +8,11 @@ type Result struct { value string } -func NewResult(value string) Result { - return Result{value} +func NewResult(value string) *Result { + return &Result{value} } -func (r Result) GetValue(ctx context.Context) (string, error) { +func (r *Result) GetValue(ctx context.Context) (string, error) { return r.value, nil } diff --git a/internal/query/types/jsonExtractor/service.go b/internal/query/types/jsonExtractor/service.go index a8ec6639..22c674aa 100644 --- a/internal/query/types/jsonExtractor/service.go +++ b/internal/query/types/jsonExtractor/service.go @@ -7,23 +7,24 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" + "queryorchestration/internal/serviceconfig" "github.com/tidwall/gjson" ) type Extractor struct { - db *database.Connection + cfg serviceconfig.ConfigProvider } type Config struct { Path string `json:"path"` } -func NewExtractor(db *database.Connection) Extractor { - return Extractor{db} +func NewExtractor(cfg serviceconfig.ConfigProvider) *Extractor { + return &Extractor{cfg} } -func (e Extractor) Process(ctx context.Context, query *resultprocessor.Query, values *[]resultprocessor.Value) (string, error) { +func (e *Extractor) Process(ctx context.Context, query *resultprocessor.Query, values *[]resultprocessor.Value) (string, error) { if values == nil || len(*values) != 1 { return "", fmt.Errorf("JSON Extraction requires 1 result") } @@ -33,7 +34,7 @@ func (e Extractor) Process(ctx context.Context, query *resultprocessor.Query, va return "", err } - byteConfig, err := e.db.Queries.GetQueryConfig(ctx, &repository.GetQueryConfigParams{ + byteConfig, err := e.cfg.GetDBQueries().GetQueryConfig(ctx, &repository.GetQueryConfigParams{ Queryid: database.MustToDBUUID(query.ID), Addedversion: query.Version, }) diff --git a/internal/query/types/jsonExtractor/updator.go b/internal/query/types/jsonExtractor/updator.go index ce8a35b5..bd2a307d 100644 --- a/internal/query/types/jsonExtractor/updator.go +++ b/internal/query/types/jsonExtractor/updator.go @@ -2,19 +2,17 @@ package jsonextractor import ( "context" - "queryorchestration/internal/database" resultprocessor "queryorchestration/internal/query/result/processor" ) type Updator struct { - db *database.Connection } -func NewUpdator(db *database.Connection) Updator { - return Updator{db} +func NewUpdator() *Updator { + return &Updator{} } -func (s Updator) Validate(ctx context.Context, current *resultprocessor.Query, entity *resultprocessor.Update) error { +func (s *Updator) Validate(ctx context.Context, current *resultprocessor.Query, entity *resultprocessor.Update) error { // TODO return nil } diff --git a/internal/query/types/jsonExtractor/updator_test.go b/internal/query/types/jsonExtractor/updator_test.go index ba5427cd..f61d99b1 100644 --- a/internal/query/types/jsonExtractor/updator_test.go +++ b/internal/query/types/jsonExtractor/updator_test.go @@ -2,10 +2,10 @@ package jsonextractor_test import ( "context" - "queryorchestration/internal/database" "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -20,13 +20,11 @@ func TestUpdatorValidate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := jsonextractor.NewUpdator(db) + svc := jsonextractor.NewUpdator() assert.NotNil(t, svc) current := &resultprocessor.Query{ diff --git a/internal/query/update.go b/internal/query/update.go index cca38af0..17afe35d 100644 --- a/internal/query/update.go +++ b/internal/query/update.go @@ -44,7 +44,7 @@ func (s *Service) normalizeUpdate(ctx context.Context, current *Query, entity *r } if entity.RequiredQueryIDs != nil { - createsloop, err := s.db.Queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{ + createsloop, err := s.cfg.GetDBQueries().IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{ Requiredqueryid: database.MustToDBUUID(current.ID), ID: database.MustToDBUUIDArray(*entity.RequiredQueryIDs), }) @@ -80,81 +80,73 @@ func (s *Service) normalizeUpdate(ctx context.Context, current *Query, entity *r } func (s *Service) submitUpdate(ctx context.Context, current *Query, entity *resultprocessor.Update) error { - tx, err := s.db.Pool.Begin(ctx) - if err != nil { - return err - } - defer func() { - _ = tx.Rollback(ctx) - }() + err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error { + latestVersion := current.LatestVersion + 1 + id := database.MustToDBUUID(entity.ID) - qtx := s.db.Queries.WithTx(tx) - - latestVersion := current.LatestVersion + 1 - id := database.MustToDBUUID(entity.ID) - - addIDs := getSetDifference(entity.RequiredQueryIDs, current.RequiredQueryIDs) - for _, qID := range addIDs { - err = qtx.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ - Queryid: id, - Requiredqueryid: database.MustToDBUUID(qID), - Addedversion: latestVersion, - }) - if err != nil { - return err + addIDs := getSetDifference(entity.RequiredQueryIDs, current.RequiredQueryIDs) + for _, qID := range addIDs { + err := q.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ + Queryid: id, + Requiredqueryid: database.MustToDBUUID(qID), + Addedversion: latestVersion, + }) + if err != nil { + return err + } } - } - removeIDs := getSetDifference(current.RequiredQueryIDs, entity.RequiredQueryIDs) - for _, qID := range removeIDs { - err = qtx.RemoveRequiredQuery(ctx, &repository.RemoveRequiredQueryParams{ - Queryid: id, - Requiredqueryid: database.MustToDBUUID(qID), - Removedversion: &latestVersion, - }) - if err != nil { - return err + removeIDs := getSetDifference(current.RequiredQueryIDs, entity.RequiredQueryIDs) + for _, qID := range removeIDs { + err := q.RemoveRequiredQuery(ctx, &repository.RemoveRequiredQueryParams{ + Queryid: id, + Requiredqueryid: database.MustToDBUUID(qID), + Removedversion: &latestVersion, + }) + if err != nil { + return err + } } - } - if entity.Config != nil && *entity.Config != "" { - err = qtx.RemoveQueryConfig(ctx, &repository.RemoveQueryConfigParams{ - Queryid: id, - Removedversion: &latestVersion, + if entity.Config != nil && *entity.Config != "" { + err := q.RemoveQueryConfig(ctx, &repository.RemoveQueryConfigParams{ + Queryid: id, + Removedversion: &latestVersion, + }) + if err != nil { + return err + } + + err = q.AddQueryConfig(ctx, &repository.AddQueryConfigParams{ + Queryid: id, + Config: []byte(*entity.Config), + Addedversion: latestVersion, + }) + if err != nil { + return err + } + } + + activeVersion := entity.ActiveVersion + if activeVersion == nil { + activeVersion = ¤t.ActiveVersion + } + + err := q.UpdateQuery(ctx, &repository.UpdateQueryParams{ + Latestversion: latestVersion, + Activeversion: *activeVersion, + ID: id, }) if err != nil { return err } - err = qtx.AddQueryConfig(ctx, &repository.AddQueryConfigParams{ - Queryid: id, - Config: []byte(*entity.Config), - Addedversion: latestVersion, - }) - if err != nil { - return err - } - } - - activeVersion := entity.ActiveVersion - if activeVersion == nil { - activeVersion = ¤t.ActiveVersion - } - - err = qtx.UpdateQuery(ctx, &repository.UpdateQueryParams{ - Latestversion: latestVersion, - Activeversion: *activeVersion, - ID: id, + return nil }) if err != nil { return err } - err = tx.Commit(ctx) - if err != nil { - return err - } - return nil } @@ -185,9 +177,9 @@ func getSetDifference(setA *[]uuid.UUID, setB *[]uuid.UUID) []uuid.UUID { func (s *Service) getUpdator(qType resultprocessor.Type) (resultprocessor.Updator, error) { switch qType { case resultprocessor.TypeJsonExtractor: - return jsonextractor.NewUpdator(s.db), nil + return jsonextractor.NewUpdator(), nil case resultprocessor.TypeContextFull: - return contextfull.NewUpdator(s.db), nil + return contextfull.NewUpdator(), nil default: return nil, fmt.Errorf("attempting to process invalid query type") } diff --git a/internal/query/update_test.go b/internal/query/update_test.go index 654404fb..02d8381d 100644 --- a/internal/query/update_test.go +++ b/internal/query/update_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/database/repository" "queryorchestration/internal/query" resultprocessor "queryorchestration/internal/query/result/processor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -21,12 +22,10 @@ func TestUpdate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := query.New(db, &query.Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := query.New(cfg, &query.Services{}) config := "{\"path\":\"example_path\"}" existing := query.Query{ diff --git a/internal/query/updateprivate_test.go b/internal/query/updateprivate_test.go index 558b4bb7..1f40e589 100644 --- a/internal/query/updateprivate_test.go +++ b/internal/query/updateprivate_test.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -19,12 +20,10 @@ func TestGetUpdator(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := New(db, &Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg, &Services{}) queryType := resultprocessor.Type(resultprocessor.TypeContextFull) updator, err := svc.getUpdator(queryType) @@ -48,12 +47,10 @@ func TestSubmitUpdate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := New(db, &Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg, &Services{}) config := "{\"path\":\"example_path\"}" q := Query{ @@ -99,12 +96,10 @@ func TestSubmitUpdateRollback(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := New(db, &Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg, &Services{}) q := Query{ ID: uuid.New(), @@ -134,12 +129,10 @@ func TestSubmitUpdateRequiredQueries(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := New(db, &Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg, &Services{}) config := "{\"path\":\"example_path\"}" q := Query{ @@ -186,12 +179,10 @@ func TestSubmitUpdateActiveVersion(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := New(db, &Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg, &Services{}) q := Query{ ID: uuid.New(), @@ -233,12 +224,10 @@ func TestNormalizeUpdate(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - svc := New(db, &Services{}) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg, &Services{}) current := &Query{ ID: uuid.New(), @@ -246,11 +235,11 @@ func TestNormalizeUpdate(t *testing.T) { LatestVersion: int32(2), Type: resultprocessor.TypeJsonExtractor, } - cfg := "{}" + qcfg := "{}" aV := int32(2) update := &resultprocessor.Update{ ID: current.ID, - Config: &cfg, + Config: &qcfg, ActiveVersion: &aV, RequiredQueryIDs: &[]uuid.UUID{}, } @@ -267,7 +256,7 @@ func TestNormalizeUpdate(t *testing.T) { assert.EqualExportedValues(t, resultprocessor.Update{ ID: current.ID, ActiveVersion: &aV, - Config: &cfg, + Config: &qcfg, RequiredQueryIDs: nil, }, *update) diff --git a/internal/server/api/listener.go b/internal/server/api/listener.go deleted file mode 100644 index d1ca176b..00000000 --- a/internal/server/api/listener.go +++ /dev/null @@ -1,94 +0,0 @@ -package api - -import ( - "context" - "log" - "net" - "net/http" - "queryorchestration/internal/server" - "strconv" - - "github.com/getkin/kin-openapi/openapi3" - "github.com/labstack/echo/v4" - "github.com/labstack/echo/v4/middleware" - _ "github.com/lib/pq" - echoSwagger "github.com/swaggo/echo-swagger" - "gopkg.in/yaml.v3" -) - -type Config struct { - RegisterHandlers func(*server.Config, *echo.Echo) *APIConfig - BasePath string -} - -type APIConfig struct { - Swagger *openapi3.T -} - -type Server struct { - port int - host string - address string - server *echo.Echo -} - -func New(ctx context.Context, cfg *Config) *Server { - serverCfg := server.New(ctx, &server.NewConfig{ - BasePath: cfg.BasePath, - }) - - e := echo.New() - - e.Use(middleware.Logger()) - e.Use(middleware.Recover()) - e.Use(middleware.CORS()) - - apicfg := cfg.RegisterHandlers(serverCfg, e) - - e.GET("/swagger/doc.json", func(c echo.Context) error { - spec, err := apicfg.Swagger.MarshalJSON() - if err != nil { - return err - } - return c.Blob(http.StatusOK, "application/json", spec) - }) - e.GET("/swagger/doc.yaml", func(c echo.Context) error { - spec, err := apicfg.Swagger.MarshalYAML() - if err != nil { - return err - } - yaml, err := yaml.Marshal(spec) - if err != nil { - return err - } - - return c.Blob(http.StatusOK, "application/yaml", yaml) - }) - - e.GET("/swagger/*", echoSwagger.EchoWrapHandler( - echoSwagger.DocExpansion("full"), - echoSwagger.DeepLinking(true), - echoSwagger.DomID("swagger-ui"), - echoSwagger.PersistAuthorization(true), - )) - - host := "0.0.0.0" - port := 8080 - address := net.JoinHostPort(host, strconv.Itoa(port)) - - return &Server{ - port: port, - host: host, - address: address, - server: e, - } -} - -func (s *Server) Listen() { - log.Printf("Listening on port %d", s.port) - - err := s.server.Start(s.address) - if err != nil { - s.server.Logger.Panic(err) - } -} diff --git a/internal/server/api/listener_test.go b/internal/server/api/listener_test.go deleted file mode 100644 index ee002f7e..00000000 --- a/internal/server/api/listener_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package api - -import ( - "context" - "os" - "path" - "queryorchestration/internal/database" - "queryorchestration/internal/server" - "queryorchestration/internal/test" - "testing" - - "github.com/labstack/echo/v4" - "github.com/stretchr/testify/assert" -) - -func TestNew(t *testing.T) { - if testing.Short() { - t.Skip("Skipping long test in short mode") - } - ctx := context.Background() - - _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../../.."), - }, - }) - defer cleanup() - - registerHandlers := func(c *server.Config, e *echo.Echo) *APIConfig { - return nil - } - cfg := &Config{ - RegisterHandlers: registerHandlers, - BasePath: path.Join(os.Getenv("PWD"), "../../.."), - } - - server := New(ctx, cfg) - assert.Equal(t, 8080, server.port) - assert.Equal(t, "0.0.0.0", server.host) - assert.Equal(t, "0.0.0.0:8080", server.address) -} - -func TestListen(t *testing.T) { - server := Server{} - - assert.Panics(t, func() { server.Listen() }) -} diff --git a/internal/server/env/get.go b/internal/server/env/get.go deleted file mode 100644 index 1eb2aa68..00000000 --- a/internal/server/env/get.go +++ /dev/null @@ -1,25 +0,0 @@ -package env - -import ( - "fmt" - "log" - "os" -) - -func GetPanic(name string) string { - value, err := Get(name) - if err != nil { - log.Panic(err.Error()) - } - - return value -} - -func Get(name string) (string, error) { - value, exists := os.LookupEnv(name) - if !exists || value == "" { - return "", fmt.Errorf("environment variable not set: %s", name) - } - - return value, nil -} diff --git a/internal/server/env/get_test.go b/internal/server/env/get_test.go deleted file mode 100644 index 7bb1489c..00000000 --- a/internal/server/env/get_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package env_test - -import ( - "queryorchestration/internal/server/env" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestGet(t *testing.T) { - name := "test_get_env_var_does_not_exist" - - value := "value" - t.Setenv(name, value) - - returnValue, err := env.Get(name) - - assert.Equal(t, value, returnValue) - assert.Nil(t, err) - - value = "" - t.Setenv(name, value) - - returnValue, err = env.Get(name) - - assert.Equal(t, value, returnValue) - assert.NotNil(t, err) -} diff --git a/internal/server/env/getfatal_test.go b/internal/server/env/getfatal_test.go deleted file mode 100644 index 583ef895..00000000 --- a/internal/server/env/getfatal_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package env_test - -import ( - "queryorchestration/internal/server/env" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestGetPanic(t *testing.T) { - name := "test_get_env_var_does_not_exist" - - value := "value" - t.Setenv(name, value) - - returnValue := env.GetPanic(name) - - assert.Equal(t, value, returnValue) - - value = "" - t.Setenv(name, value) - - assert.Panics(t, func() { env.GetPanic(name) }) -} diff --git a/internal/server/otel/service.go b/internal/server/otel/service.go index 845753a4..ddb562d6 100644 --- a/internal/server/otel/service.go +++ b/internal/server/otel/service.go @@ -2,23 +2,23 @@ package otel import ( "context" - "log" - "os" + "log/slog" + "queryorchestration/internal/serviceconfig" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" sdktrace "go.opentelemetry.io/otel/sdk/trace" ) -func New(ctx context.Context) func() { - if os.Getenv("ENABLE_OTEL") != "1" { - log.Println("OpenTelemetry is disabled. Set ENABLE_OTEL to enable.") - return func() {} +func New(ctx context.Context, cfg serviceconfig.ObservabilityConfigProvider) func() error { + if !cfg.IsOtelEnabled() { + slog.Warn("OpenTelemetry is disabled. Set ENABLE_OTEL to true to enable.") + return func() error { return nil } } exporter, err := otlptracegrpc.New(ctx) if err != nil { - log.Panic(err) + return func() error { return nil } } tp := sdktrace.NewTracerProvider( @@ -27,10 +27,11 @@ func New(ctx context.Context) func() { otel.SetTracerProvider(tp) - return func() { + return func() error { err := tp.Shutdown(ctx) if err != nil { - log.Panic(err) + return err } + return nil } } diff --git a/internal/server/otel/service_test.go b/internal/server/otel/service_test.go index f8f7dca7..26d8266b 100644 --- a/internal/server/otel/service_test.go +++ b/internal/server/otel/service_test.go @@ -2,7 +2,9 @@ package otel_test import ( "context" + "fmt" "queryorchestration/internal/server/otel" + "queryorchestration/internal/serviceconfig" "testing" "github.com/stretchr/testify/assert" @@ -11,6 +13,15 @@ import ( func TestNew(t *testing.T) { ctx := context.Background() - svc := otel.New(ctx) - assert.NotNil(t, svc) + cfg := &serviceconfig.ObservabilityConfig{ + EnableOtel: true, + } + clean := otel.New(ctx, cfg) + assert.NotNil(t, clean) + defer func() { + if err := clean(); err != nil { + // Log cleanup error but don't panic since we're shutting down + fmt.Printf("Error during clean: %v", err) + } + }() } diff --git a/internal/server/queue/config.go b/internal/server/queue/config.go index 9b74736e..fb1aa7e5 100644 --- a/internal/server/queue/config.go +++ b/internal/server/queue/config.go @@ -1,17 +1,10 @@ package queue import ( - "context" - "github.com/aws/aws-sdk-go-v2/service/sqs" - "github.com/aws/aws-sdk-go-v2/service/sqs/types" ) type Config struct { URL string Client *sqs.Client } - -type Controller interface { - Process(ctx context.Context, message *types.Message) error -} diff --git a/internal/server/queue/listener.go b/internal/server/queue/listener.go deleted file mode 100644 index 24bd8f1d..00000000 --- a/internal/server/queue/listener.go +++ /dev/null @@ -1,54 +0,0 @@ -package queue - -import ( - "context" - "log" - "queryorchestration/internal/server" - "queryorchestration/internal/server/env" - - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/sqs" -) - -type ListenerConfig struct { - Controller func(*server.Config) Controller - BasePath string -} - -type Server struct { - controller Controller - queueConnection *Config -} - -func NewServer(ctx context.Context, lConfig *ListenerConfig) *Server { - serverCfg := server.New(ctx, &server.NewConfig{ - BasePath: lConfig.BasePath, - }) - - cfg, err := config.LoadDefaultConfig(ctx) - if err != nil { - log.Panicf("Unable to load SDK config: %v", err) - } - - queueURL := env.GetPanic("QUEUE_URL") - - sqsClient := sqs.NewFromConfig(cfg) - - return &Server{ - controller: lConfig.Controller(serverCfg), - queueConnection: &Config{ - URL: queueURL, - Client: sqsClient, - }, - } -} - -func (s *Server) Listen(ctx context.Context) { - config := &PollConfig{ - Controller: s.controller, - Config: s.queueConnection, - } - - log.Print("Listening to queue") - PollMessages(ctx, config) -} diff --git a/internal/server/queue/listener_test.go b/internal/server/queue/listener_test.go deleted file mode 100644 index 46ffcc79..00000000 --- a/internal/server/queue/listener_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package queue_test - -import ( - "context" - "os" - "path" - "queryorchestration/internal/server" - "queryorchestration/internal/server/queue" - "queryorchestration/internal/test" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestNew(t *testing.T) { - if testing.Short() { - t.Skip("Skipping long test in short mode") - } - ctx := context.Background() - _, cleanup := test.CreateQueue(t, ctx, &test.CreateQueueConfig{}) - defer cleanup() - _, cleanup = test.CreateDB(t, ctx, &test.CreateDatabaseConfig{}) - defer cleanup() - - ctx, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - - controller := func(cfg *server.Config) queue.Controller { - return MockController{} - } - - t.Setenv("QUEUE_URL", "example_url") - - queue.NewServer(ctx, &queue.ListenerConfig{ - Controller: controller, - BasePath: path.Join(os.Getenv("PWD"), "../../.."), - }) -} - -func TestListen(t *testing.T) { - ctx := context.Background() - - queue := queue.Server{} - - assert.Panics(t, func() { queue.Listen(ctx) }) -} diff --git a/internal/server/queue/poll.go b/internal/server/queue/poll.go deleted file mode 100644 index 4bff112d..00000000 --- a/internal/server/queue/poll.go +++ /dev/null @@ -1,51 +0,0 @@ -package queue - -import ( - "context" - "fmt" - "log" -) - -type PollConfig struct { - Config *Config - Controller Controller -} - -func PollMessages(ctx context.Context, queueConfig *PollConfig) { - for { - select { - case <-ctx.Done(): - return - default: - err := PollMessage(ctx, queueConfig) - if err != nil { - log.Print(err) - } - } - } -} - -func PollMessage(ctx context.Context, queueConfig *PollConfig) error { - result, err := Receive(ctx, queueConfig.Config, []string{ - "type", - }) - if err != nil { - return fmt.Errorf("message fetch fail: %v", err) - } - - for _, message := range result.Messages { - go func() { - err := queueConfig.Controller.Process(ctx, &message) - if err != nil { - log.Printf("message process fail: %v", err) - } - - err = Delete(ctx, queueConfig.Config, &message) - if err != nil { - log.Printf("message delete fail: %v", err) - } - }() - } - - return nil -} diff --git a/internal/server/runner/listener.go b/internal/server/runner/listener.go new file mode 100644 index 00000000..84f5f96c --- /dev/null +++ b/internal/server/runner/listener.go @@ -0,0 +1,60 @@ +package runner + +import ( + "context" + "fmt" + "queryorchestration/internal/server" + "queryorchestration/internal/serviceconfig/aws" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" +) + +type Controller interface { + Process(ctx context.Context, message *types.Message) error +} + +type ListenerConfig interface { + server.Config + aws.ConfigProvider + RegisterController() Controller + GetQueueURL() string +} + +type BaseConfig struct { + server.BaseConfig + aws.AWSConfig + ControllerFunc func() Controller + QueueURL string `env:"QUEUE_URL,required,notEmpty"` +} + +func (c *BaseConfig) RegisterController() Controller { + return c.ControllerFunc() +} +func (c *BaseConfig) GetQueueURL() string { + return c.QueueURL +} + +func New(ctx context.Context, cfg ListenerConfig) (*Server, error) { + cleanup, err := server.New(ctx, cfg) + if err != nil { + return nil, fmt.Errorf("unable run server.New: %v", err) + } + + qcfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("unable to load SDK config: %v", err) + } + + sqsClient := sqs.NewFromConfig(qcfg) + + controller := cfg.RegisterController() + + return &Server{ + controller: controller, + queueURL: cfg.GetQueueURL(), + client: sqsClient, + cleanup: cleanup, + }, nil +} diff --git a/internal/server/runner/listener_test.go b/internal/server/runner/listener_test.go new file mode 100644 index 00000000..ea0c21cc --- /dev/null +++ b/internal/server/runner/listener_test.go @@ -0,0 +1,60 @@ +package runner_test + +import ( + "context" + "os" + "path" + "queryorchestration/internal/server/runner" + "queryorchestration/internal/serviceconfig" + "queryorchestration/internal/test" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "github.com/stretchr/testify/assert" +) + +type MockController struct{} + +func (s *MockController) Process(ctx context.Context, req *types.Message) error { + return nil +} + +func TestNew(t *testing.T) { + if testing.Short() { + t.Skip("Skipping long test in short mode") + } + ctx := context.Background() + t.Setenv("QUEUE_URL", "example_url") + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &runner.BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) + + _, cleanup := test.CreateQueue(t, ctx, &test.CreateQueueConfig{}) + defer cleanup() + _, cleanup = test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) + defer cleanup() + + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + cfg.ControllerFunc = func() runner.Controller { + return &MockController{} + } + + srvPtr, err := runner.New(ctx, cfg) + assert.NotNil(t, srvPtr) + assert.NoError(t, err) +} + +func TestListen(t *testing.T) { + ctx := context.Background() + + queue := runner.Server{} + + assert.Panics(t, func() { queue.Listen(ctx) }) +} diff --git a/internal/server/runner/poll.go b/internal/server/runner/poll.go new file mode 100644 index 00000000..fe6fa187 --- /dev/null +++ b/internal/server/runner/poll.go @@ -0,0 +1,98 @@ +package runner + +import ( + "context" + "fmt" + "log/slog" + "queryorchestration/internal/server/queue" + + "github.com/aws/aws-sdk-go-v2/service/sqs" +) + +type Server struct { + controller Controller + queueURL string + client *sqs.Client + cleanup func() error +} + +func (c *Server) Listen(ctx context.Context) { + slog.Info("Listening to queue") + defer func() { + if err := c.cleanup(); err != nil { + fmt.Printf("Error during cleanup: %v", err) + } + }() + + // TODO need to slow down the polling so we are not spinning the cpu or burning up API calls. + for { + select { + case <-ctx.Done(): + return + default: + err := c.pollMessage(ctx) + if err != nil { + slog.Error(err.Error()) + } + } + } + +} + +// Alternative without go func if no pool is needed. +// Just for comment for the next function which needs to be made safe. +//func (c *Server) pollMessage(ctx context.Context) error { +// cfg := &queue.Config{ +// URL: c.queueURL, +// Client: c.client, +// } +// result, err := queue.Receive(ctx, cfg, []string{"type"}) +// if err != nil { +// return fmt.Errorf("message fetch fail: %v", err) +// } +// +// for _, message := range result.Messages { +// if err := c.controller.Process(ctx, &message); err != nil { +// return fmt.Errorf("message process fail: %v", err) +// } +// +// if err := queue.Delete(ctx, cfg, &message); err != nil { +// return fmt.Errorf("message delete fail: %v", err) +// } +// } +// +// return nil +//} + +func (c *Server) pollMessage(ctx context.Context) error { + cfg := &queue.Config{ + URL: c.queueURL, + Client: c.client, + } + result, err := queue.Receive(ctx, cfg, []string{ + "type", + }) + if err != nil { + return fmt.Errorf("message fetch fail: %v", err) + } + + for _, message := range result.Messages { + // TODO Need to remove this go func() !! + // Must not have uncontrolled go routine creation. See commented example above if + // simple case. + // if process takes too long to complete then we need to use a thread pool. + go func() { + err := c.controller.Process(ctx, &message) + if err != nil { + slog.Error("message process fail", "err", err) + } + + err = queue.Delete(ctx, cfg, &message) + if err != nil { + slog.Error("message delete fail", "err", err) + } + }() + } + + return nil +} diff --git a/internal/server/queue/poll_test.go b/internal/server/runner/poll_test.go similarity index 62% rename from internal/server/queue/poll_test.go rename to internal/server/runner/poll_test.go index c9616f33..8b3dbd5f 100644 --- a/internal/server/queue/poll_test.go +++ b/internal/server/runner/poll_test.go @@ -1,8 +1,7 @@ -package queue_test +package runner import ( "context" - "queryorchestration/internal/server/queue" "queryorchestration/internal/test" "testing" "time" @@ -13,7 +12,7 @@ import ( type MockController struct{} -func (s MockController) Process(ctx context.Context, req *types.Message) error { +func (s *MockController) Process(ctx context.Context, req *types.Message) error { return nil } @@ -30,14 +29,13 @@ func TestPollMessages(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() - cfg := &queue.Config{ - URL: queueConfig.URL, - Client: queueConfig.Client, + cfg := &Server{ + controller: &controller, + queueURL: queueConfig.URL, + client: queueConfig.Client, + cleanup: func() error { return nil }, } - queue.PollMessages(ctx, &queue.PollConfig{ - Config: cfg, - Controller: controller, - }) + cfg.Listen(ctx) } func TestPollMessage(t *testing.T) { @@ -50,13 +48,11 @@ func TestPollMessage(t *testing.T) { controller := MockController{} - cfg := &queue.Config{ - URL: queueConfig.URL, - Client: queueConfig.Client, + cfg := &Server{ + controller: &controller, + queueURL: queueConfig.URL, + client: queueConfig.Client, } - err := queue.PollMessage(ctx, &queue.PollConfig{ - Config: cfg, - Controller: controller, - }) + err := cfg.pollMessage(ctx) assert.Nil(t, err) } diff --git a/internal/server/server.go b/internal/server/server.go index 2412662e..26486476 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -2,47 +2,47 @@ package server import ( "context" - "log" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" + "queryorchestration/internal/database/migrations" "queryorchestration/internal/server/otel" + "queryorchestration/internal/serviceconfig" "github.com/go-playground/validator/v10" ) -type Config struct { - Database *database.Connection +type Config interface { + serviceconfig.ConfigProvider + SetValidator() + GetValidator() *validator.Validate +} + +type BaseConfig struct { + serviceconfig.BaseConfig Validator *validator.Validate } -type NewConfig struct { - BasePath string -} - -func New(ctx context.Context, cfg *NewConfig) *Config { - closeTracer := otel.New(ctx) - defer closeTracer() - - database.RunMigrations(ctx, &database.MigrationConfig{ - BasePath: cfg.BasePath, - }) - +func (c *BaseConfig) SetValidator() { valid := validator.New() - dbPool := database.GetDBPool(ctx) - dbQueries := repository.New(dbPool) - db := &database.Connection{ - Pool: dbPool, - Queries: dbQueries, - } - - err := dbPool.Ping(ctx) - if err != nil { - log.Panic("Unable to ping database") - } - - return &Config{ - Database: db, - Validator: valid, - } + c.Validator = valid +} +func (c *BaseConfig) GetValidator() *validator.Validate { + return c.Validator +} + +func New(ctx context.Context, cfg Config) (func() error, error) { + closeTracer := otel.New(ctx, cfg) + + err := migrations.Run(ctx, cfg) + if err != nil { + return nil, err + } + + cfg.SetValidator() + + err = cfg.SetDBPool(ctx) + if err != nil { + return nil, err + } + + return closeTracer, nil } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index cc9edd17..b9415116 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -2,6 +2,7 @@ package server_test import ( "context" + "fmt" "os" "path" "queryorchestration/internal/server" @@ -17,13 +18,20 @@ func TestNew(t *testing.T) { } ctx := context.Background() - _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{}) + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../..")) + cfg := &server.BaseConfig{} + + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + }) defer cleanup() - newCfg := &server.NewConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../.."), - } - - cfg := server.New(ctx, newCfg) - assert.NotNil(t, cfg) + clean, err := server.New(ctx, cfg) + assert.NoError(t, err) + defer func() { + if err := clean(); err != nil { + // Log cleanup error but don't panic since we're shutting down + fmt.Printf("Error during clean: %v", err) + } + }() } diff --git a/internal/server/service/listener.go b/internal/server/service/listener.go new file mode 100644 index 00000000..84a999fd --- /dev/null +++ b/internal/server/service/listener.go @@ -0,0 +1,128 @@ +package service + +import ( + "context" + "log/slog" + "net" + "net/http" + "queryorchestration/internal/server" + "strconv" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + _ "github.com/lib/pq" + echoSwagger "github.com/swaggo/echo-swagger" + "gopkg.in/yaml.v3" +) + +type Config interface { + server.Config + SetRouter(*echo.Echo) + GetRouter() *echo.Echo + RegisterHandlers() (*openapi3.T, error) +} + +type BaseConfig struct { + server.BaseConfig + RegisterHandlersFunc func() (*openapi3.T, error) + OpenAPI *openapi3.T + Router *echo.Echo +} + +func (c *BaseConfig) SetRouter(e *echo.Echo) { + c.Router = e +} +func (c *BaseConfig) GetRouter() *echo.Echo { + return c.Router +} +func (c *BaseConfig) RegisterHandlers() (*openapi3.T, error) { + o, err := c.RegisterHandlersFunc() + if err != nil { + return nil, err + } + c.OpenAPI = o + return c.OpenAPI, nil +} + +type Server struct { + port int + host string + address string + router *echo.Echo + cleanup func() error +} + +func New(ctx context.Context, cfg Config) (*Server, error) { + cleanup, err := server.New(ctx, cfg) + if err != nil { + return nil, err + } + + e := echo.New() + + e.Use(middleware.Logger()) + e.Use(middleware.Recover()) + e.Use(middleware.CORS()) + + cfg.SetRouter(e) + + opnapi, err := cfg.RegisterHandlers() + if err != nil { + return nil, err + } + + e.GET("/swagger/doc.json", func(c echo.Context) error { + spec, err := opnapi.MarshalJSON() + if err != nil { + return err + } + return c.Blob(http.StatusOK, "application/json", spec) + }) + e.GET("/swagger/doc.yaml", func(c echo.Context) error { + spec, err := opnapi.MarshalYAML() + if err != nil { + return err + } + yaml, err := yaml.Marshal(spec) + if err != nil { + return err + } + + return c.Blob(http.StatusOK, "application/yaml", yaml) + }) + + e.GET("/swagger/*", echoSwagger.EchoWrapHandler( + echoSwagger.DocExpansion("full"), + echoSwagger.DeepLinking(true), + echoSwagger.DomID("swagger-ui"), + echoSwagger.PersistAuthorization(true), + )) + + host := "0.0.0.0" + port := 8080 + address := net.JoinHostPort(host, strconv.Itoa(port)) + + return &Server{ + port: port, + host: host, + address: address, + router: e, + cleanup: cleanup, + }, nil +} + +func (s *Server) Listen() { + slog.Info("Listening for requests", "port", s.port) + defer func() { + if err := s.cleanup(); err != nil { + // Log cleanup error but don't panic since we're shutting down + s.router.Logger.Errorf("Error during cleanup: %v", err) + } + }() + + err := s.router.Start(s.address) + if err != nil { + s.router.Logger.Panic(err) + } +} diff --git a/internal/server/service/listener_test.go b/internal/server/service/listener_test.go new file mode 100644 index 00000000..f4d1623e --- /dev/null +++ b/internal/server/service/listener_test.go @@ -0,0 +1,50 @@ +package service + +import ( + "context" + "os" + "path" + "queryorchestration/internal/serviceconfig" + + "queryorchestration/internal/test" + "testing" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/stretchr/testify/assert" +) + +func TestNew(t *testing.T) { + if testing.Short() { + t.Skip("Skipping long test in short mode") + } + ctx := context.Background() + + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &BaseConfig{} + err := serviceconfig.InitializeConfig(cfg) + assert.Nil(t, err) + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) + defer cleanup() + + registerHandlers := func() (*openapi3.T, error) { + return nil, nil + } + cfg.RegisterHandlersFunc = registerHandlers + errInitializingConfig := serviceconfig.InitializeConfig(cfg) + assert.NoError(t, errInitializingConfig) + + serverInstance, errCreatingServer := New(ctx, cfg) + assert.NoError(t, errCreatingServer) + assert.Equal(t, 8080, serverInstance.port) + assert.Equal(t, "0.0.0.0", serverInstance.host) + assert.Equal(t, "0.0.0.0:8080", serverInstance.address) +} + +func TestListen(t *testing.T) { + server := Server{} + + assert.Panics(t, func() { server.Listen() }) +} diff --git a/internal/serviceconfig/aws/config.go b/internal/serviceconfig/aws/config.go new file mode 100644 index 00000000..d3d9d3a0 --- /dev/null +++ b/internal/serviceconfig/aws/config.go @@ -0,0 +1,8 @@ +package aws + +type AWSConfig struct { + AWSClientID string `env:"AWS_CLIENT_ID"` // AWS client identifier + AWSSecretKey string `env:"AWS_SECRET_KEY"` // AWS secret access key +} + +type ConfigProvider interface{} diff --git a/internal/serviceconfig/common.go b/internal/serviceconfig/common.go index 4207aab3..db5adf10 100644 --- a/internal/serviceconfig/common.go +++ b/internal/serviceconfig/common.go @@ -1,46 +1,51 @@ package serviceconfig import ( + "errors" "fmt" - "github.com/caarlos0/env/v11" - "github.com/joho/godotenv" "log/slog" "os" + "queryorchestration/internal/serviceconfig/database" "reflect" "strings" + + "github.com/caarlos0/env/v11" + "github.com/joho/godotenv" ) // Common Types // ------------ -// DbConfig contains database connection configuration parameters -// that are shared across services. -type DbConfig struct { - DBUser string `env:"DB_USER,required,notEmpty"` // Database username - DBSecret string `env:"DB_PASS,required,notEmpty"` // Database password - DBHost string `env:"DB_HOST,required,notEmpty"` // Database host address - DBPort int `env:"DB_PORT,required,notEmpty"` // Database port number - DBName string `env:"DB_NAME,required,notEmpty"` // Database name - DBNoSSL bool `env:"DB_NOSSL" envDefault:"false"` // SSL mode configuration -} - -// LoggerConfig wraps the application logger instance. +// Logging type LoggerConfig struct { - Logger *slog.Logger // Structured logger instance + Logger *slog.Logger } -// AwsConfig contains AWS credentials and configuration. -type AwsConfig struct { - AWSClientID string `env:"AWS_CLIENT_ID"` // AWS client identifier - AWSSecretKey string `env:"AWS_SECRET_KEY"` // AWS secret access key +// Observability +type ObservabilityConfig struct { + EnableOtel bool `env:"ENABLE_OTEL" envDefault:"false"` +} + +type ObservabilityConfigProvider interface { + IsOtelEnabled() bool +} + +func (o *ObservabilityConfig) IsOtelEnabled() bool { + return o.EnableOtel } // BaseConfig provides common configuration fields and functionality // that can be embedded in service-specific configs. type BaseConfig struct { LoggerConfig - DbConfig - AwsConfig + ObservabilityConfig + database.BaseConfig + + // miscellaneous fields uncategorized + // PWD will replace the BASE_PATH env var + Pwd string `env:"PWD,required,notEmpty"` + // BASE_PATH will override the PWD env var if present + BasePath string `env:"BASE_PATH"` } // Interfaces @@ -49,6 +54,9 @@ type BaseConfig struct { // ConfigProvider defines the basic requirements for all configuration types. // Implementations must provide logging capabilities and configuration display. type ConfigProvider interface { + database.ConfigProvider + ObservabilityConfigProvider + GetBasePath() string GetLogger() *slog.Logger LogConfig(prefixSecret string) } @@ -59,33 +67,49 @@ type ConfigProvider interface { // InitializeConfig sets up a configuration instance by loading environment // variables and initializing the logger. It expects a ConfigProvider implementation. // Returns an error if initialization fails. -func InitializeConfig(cfg ConfigProvider) error { - // First, initialize the logger by getting the base config through reflection +func getBaseConfig(cfg ConfigProvider) *BaseConfig { + if cfg == nil { + return nil + } + + t := reflect.ValueOf(cfg).Type() + + if t == reflect.TypeOf(&BaseConfig{}) { + v := reflect.ValueOf(cfg).Interface().(*BaseConfig) + return v + } else if t.Kind() == reflect.Ptr && t.Elem().Kind() != reflect.Struct { + return nil + } + val := reflect.ValueOf(cfg).Elem() - // Look for the embedded BaseConfig var baseConfig *BaseConfig - for i := 0; i < val.NumField(); i++ { - if val.Type().Field(i).Anonymous && val.Field(i).Type() == reflect.TypeOf(BaseConfig{}) { - baseConfig = val.Field(i).Addr().Interface().(*BaseConfig) - break + for i := 0; i < val.NumField() && baseConfig == nil; i++ { + b := val.Field(i) + intType := reflect.TypeOf((*ConfigProvider)(nil)).Elem() + if b.Type().Implements(intType) { + baseConfig = getBaseConfig(b.Interface().(ConfigProvider)) + } else if reflect.PointerTo(b.Type()).Implements(intType) { + baseConfig = getBaseConfig(b.Addr().Interface().(ConfigProvider)) } } + return baseConfig +} + +func InitializeConfig(cfg ConfigProvider) error { + baseConfig := getBaseConfig(cfg) if baseConfig == nil { - return fmt.Errorf("no BaseConfig found in the provided config struct") + return errors.New("no BaseConfig found in the provided config struct") } - // Initialize the logger in the base config baseConfig.Logger = slog.New(slog.NewTextHandler(os.Stdout, nil)) slog.SetDefault(baseConfig.Logger) - // Load .env file if err := godotenv.Load(); err != nil { slog.Warn("No .env file found or error loading it", "error", err) } - // Parse environment variables if err := env.Parse(cfg); err != nil { if envErr, ok := err.(env.AggregateError); ok { slog.Error("Missing required environment variables:") @@ -95,16 +119,20 @@ func InitializeConfig(cfg ConfigProvider) error { } else { slog.Error("Error parsing environment variables", "error", err) } + return err } - // Log the configuration - cfg.LogConfig("secret") return nil } -// GetLogger implements the ConfigProvider interface by returning -// the configured logger instance. +func (b *BaseConfig) GetBasePath() string { + if b.BasePath != "" { + return b.BasePath + } + return b.Pwd +} + func (b *BaseConfig) GetLogger() *slog.Logger { return b.Logger } diff --git a/internal/serviceconfig/service_config_test.go b/internal/serviceconfig/common_test.go similarity index 70% rename from internal/serviceconfig/service_config_test.go rename to internal/serviceconfig/common_test.go index 5bdb9e0a..ec927404 100644 --- a/internal/serviceconfig/service_config_test.go +++ b/internal/serviceconfig/common_test.go @@ -1,10 +1,13 @@ package serviceconfig import ( - "github.com/stretchr/testify/assert" + "log/slog" "os" + "reflect" "strings" "testing" + + "github.com/stretchr/testify/assert" ) type TestingServiceNameConfig struct { @@ -35,6 +38,7 @@ func TestInitializeConfig(t *testing.T) { "DB_NOSSL": "true", "SUB_FIELD1:": "value1", "SUB_FIELD2": "42", + "PWD": "/foo", }, wantErr: false, }, @@ -49,6 +53,7 @@ func TestInitializeConfig(t *testing.T) { "DB_PORT": "5432", "DB_NAME": "query_orchestration", "DB_NOSSL": "true", + "PWD": "/foo", // INT_TEST intentionally omitted }, wantErr: true, @@ -59,6 +64,7 @@ func TestInitializeConfig(t *testing.T) { envVars: map[string]string{ "APP_ENV": "testing", "BOOL_TEST": "notabool", + "PWD": "/foo", "INT_TEST": "42", }, wantErr: true, @@ -69,6 +75,7 @@ func TestInitializeConfig(t *testing.T) { envVars: map[string]string{ "APP_ENV": "testing", "BOOL_TEST": "true", + "PWD": "/foo", "INT_TEST": "notanint", }, wantErr: true, @@ -78,12 +85,10 @@ func TestInitializeConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Clear environment before each test os.Clearenv() - // Set environment variables for the test for k, v := range tt.envVars { - os.Setenv(k, v) + t.Setenv(k, v) } cfg := &TestingServiceNameConfig{} @@ -124,3 +129,45 @@ func TestInitializeConfig(t *testing.T) { }) } } + +func TestGetBaseConfig(t *testing.T) { + assert.NotNil(t, getBaseConfig(&BaseConfig{})) + assert.Nil(t, getBaseConfig(nil)) + assert.NotNil(t, getBaseConfig(&TestingServiceNameConfig{})) +} + +func TestGetBasePath(t *testing.T) { + cfg := &BaseConfig{ + Pwd: "pwd_path", + } + assert.Equal(t, "pwd_path", cfg.GetBasePath()) + cfg.BasePath = "base_path" + assert.Equal(t, "base_path", cfg.GetBasePath()) +} + +func TestGetLogger(t *testing.T) { + cfg := &BaseConfig{} + assert.Nil(t, cfg.GetLogger()) + cfg.Logger = slog.Default() + assert.NotNil(t, cfg.GetLogger()) +} + +func TestLogConfig(t *testing.T) { + cfg := &BaseConfig{} + tl := &testLogger{T: t} + cfg.Logger = slog.New(tl) + cfg.LogConfig("") + assert.Len(t, tl.Logs, 14) + assert.Equal(t, "Logger", tl.Logs[0]["key"]) +} + +func TestLogConfigRecursive(t *testing.T) { + cfg := &BaseConfig{} + tl := &testLogger{T: t} + cfg.Logger = slog.New(tl) + v := struct{ Example string }{Example: "examplestring"} + cfg.logConfigRecursive(reflect.ValueOf(v), "", "", make(map[reflect.Value]bool)) + assert.Len(t, tl.Logs, 1) + assert.Equal(t, "Example", tl.Logs[0]["key"]) + assert.Equal(t, "examp...", tl.Logs[0]["value"]) +} diff --git a/internal/serviceconfig/database/config.go b/internal/serviceconfig/database/config.go new file mode 100644 index 00000000..4033a9c9 --- /dev/null +++ b/internal/serviceconfig/database/config.go @@ -0,0 +1,78 @@ +package database + +import ( + "context" + "fmt" + "queryorchestration/internal/database/repository" + + "github.com/jackc/pgx/v5/pgxpool" +) + +type BaseConfig struct { + DBUser string `env:"DB_USER,required,notEmpty"` + DBSecret string `env:"DB_PASS,required,notEmpty"` + DBHost string `env:"DB_HOST,required,notEmpty"` + DBPort int `env:"DB_PORT,required,notEmpty"` + DBName string `env:"DB_NAME,required,notEmpty"` + DBNoSSL bool `env:"DB_NOSSL" envDefault:"false"` + DBPool Pool + DBQueries *repository.Queries + DBPoolConfig *pgxpool.Config + DBEnumTypes []string +} + +type ConfigProvider interface { + GetDBBaseURI() string + GetDBURI() string + GetDBOpts() map[string]string + GetDBOptsString() string + GetDBName() string + GetDBDriver() string + SetDBPoolConfig() error + SetDBPool(ctx context.Context) error + GetDBQueries() *repository.Queries + GetDBPool() Pool + ExecuteDBTransaction(context.Context, func(context.Context, *repository.Queries) error) error +} + +func (b *BaseConfig) GetDBOpts() map[string]string { + opts := make(map[string]string) + + if b.DBNoSSL { + opts["sslmode"] = "disable" + } + + return opts +} + +func (b *BaseConfig) GetDBOptsString() string { + str := "" + opts := b.GetDBOpts() + isFirst := true + for key, value := range opts { + if !isFirst { + str += "&" + } else { + isFirst = false + } + str += fmt.Sprintf("%s=%s", key, value) + } + + return str +} + +func (b *BaseConfig) GetDBDriver() string { + return "postgres" +} + +func (b *BaseConfig) GetDBBaseURI() string { + return fmt.Sprintf("%s://%s:%s@%s:%d/", b.GetDBDriver(), b.DBUser, b.DBSecret, b.DBHost, b.DBPort) +} + +func (b *BaseConfig) GetDBName() string { + return b.DBName +} + +func (b *BaseConfig) GetDBURI() string { + return fmt.Sprintf("%s%s?%s", b.GetDBBaseURI(), b.DBName, b.GetDBOptsString()) +} diff --git a/internal/serviceconfig/database/config_test.go b/internal/serviceconfig/database/config_test.go new file mode 100644 index 00000000..ec41dca2 --- /dev/null +++ b/internal/serviceconfig/database/config_test.go @@ -0,0 +1,79 @@ +package database_test + +import ( + "queryorchestration/internal/serviceconfig/database" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetDBOpts(t *testing.T) { + cfg := database.BaseConfig{} + + opts := cfg.GetDBOpts() + assert.Equal(t, map[string]string{}, opts) + + cfg.DBNoSSL = true + opts = cfg.GetDBOpts() + assert.Equal(t, map[string]string{"sslmode": "disable"}, opts) +} + +func TestGetDBOptsString(t *testing.T) { + cfg := database.BaseConfig{} + + opts := cfg.GetDBOptsString() + assert.Equal(t, "", opts) + + cfg.DBNoSSL = true + opts = cfg.GetDBOptsString() + assert.Equal(t, "sslmode=disable", opts) +} + +func TestGetDBDriver(t *testing.T) { + cfg := database.BaseConfig{} + + driver := cfg.GetDBDriver() + assert.Equal(t, "postgres", driver) +} + +func TestGetDBBaseURI(t *testing.T) { + cfg := database.BaseConfig{} + + uri := cfg.GetDBBaseURI() + assert.Equal(t, "postgres://:@:0/", uri) + + cfg.DBHost = "host" + cfg.DBPort = 123 + cfg.DBUser = "user" + cfg.DBSecret = "pass" + uri = cfg.GetDBBaseURI() + assert.Equal(t, "postgres://user:pass@host:123/", uri) +} + +func TestGetDBURI(t *testing.T) { + cfg := database.BaseConfig{} + + uri := cfg.GetDBURI() + assert.Equal(t, "postgres://:@:0/?", uri) + + cfg.DBHost = "host" + cfg.DBPort = 123 + cfg.DBUser = "user" + cfg.DBSecret = "pass" + cfg.DBName = "name" + cfg.DBNoSSL = true + uri = cfg.GetDBURI() + assert.Equal(t, "postgres://user:pass@host:123/name?sslmode=disable", uri) +} + +func TestGetDBName(t *testing.T) { + cfg := database.BaseConfig{} + + name := cfg.GetDBName() + assert.Equal(t, "", name) + + cfg.DBName = "name" + name = cfg.GetDBName() + assert.Equal(t, "name", name) + assert.Equal(t, cfg.DBName, name) +} diff --git a/internal/serviceconfig/database/pool.go b/internal/serviceconfig/database/pool.go new file mode 100644 index 00000000..9a0367a1 --- /dev/null +++ b/internal/serviceconfig/database/pool.go @@ -0,0 +1,73 @@ +package database + +import ( + "context" + "errors" + "fmt" + "queryorchestration/internal/database/repository" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type Pool interface { + repository.DBTX + Begin(ctx context.Context) (pgx.Tx, error) + Ping(ctx context.Context) error +} + +func (b *BaseConfig) GetDBPool() Pool { + return b.DBPool +} + +func (b *BaseConfig) GetDBQueries() *repository.Queries { + return b.DBQueries +} + +func (b *BaseConfig) SetDBPoolConfig() error { + config, err := pgxpool.ParseConfig(b.GetDBURI()) + if err != nil { + return err + } + + b.DBEnumTypes = []string{ + "querytype", + } + + config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error { + for _, enumName := range b.DBEnumTypes { + dt, err := conn.LoadType(ctx, enumName) + if err != nil { + return fmt.Errorf("failed to load suitable enum type: %w", err) + } + conn.TypeMap().RegisterType(dt) + } + return nil + } + + b.DBPoolConfig = config + + return nil +} + +func (b *BaseConfig) SetDBPool(ctx context.Context) error { + err := b.SetDBPoolConfig() + if err != nil { + return err + } + + pool, err := pgxpool.NewWithConfig(ctx, b.DBPoolConfig) + if err != nil { + return err + } + + err = pool.Ping(ctx) + if err != nil { + return errors.New("unable to ping database") + } + + b.DBPool = pool + b.DBQueries = repository.New(b.DBPool) + + return nil +} diff --git a/internal/serviceconfig/database/pool_test.go b/internal/serviceconfig/database/pool_test.go new file mode 100644 index 00000000..5afee0ff --- /dev/null +++ b/internal/serviceconfig/database/pool_test.go @@ -0,0 +1,75 @@ +package database_test + +import ( + "context" + "os" + "path" + "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" + "queryorchestration/internal/serviceconfig/database" + "queryorchestration/internal/test" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/assert" +) + +func TestSetDBPool(t *testing.T) { + if testing.Short() { + t.Skip("Skipping long test in short mode") + } + ctx := context.Background() + + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../../..")) + cfg := &serviceconfig.BaseConfig{} + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) + defer cleanup() + + err := cfg.SetDBPool(ctx) + assert.NoError(t, err) + assert.NotNil(t, cfg.DBPool) +} + +func TestGetDBPoolConfig(t *testing.T) { + cfg := database.BaseConfig{} + + err := cfg.SetDBPoolConfig() + assert.Error(t, err) + assert.Nil(t, cfg.DBPoolConfig) + + cfg.DBName = "name" + cfg.DBHost = "host" + cfg.DBPort = 123 + cfg.DBUser = "user" + cfg.DBSecret = "pass" + err = cfg.SetDBPoolConfig() + assert.NoError(t, err) + assert.NotNil(t, cfg.DBPoolConfig) +} + +func TestGetDBPool(t *testing.T) { + cfg := database.BaseConfig{} + + pool := cfg.GetDBPool() + assert.Nil(t, pool) + + cfg.DBPool = &pgxpool.Pool{} + pool = cfg.GetDBPool() + assert.NotNil(t, pool) + assert.Equal(t, pool, cfg.DBPool) +} + +func TestGetDBQueries(t *testing.T) { + cfg := database.BaseConfig{} + + queries := cfg.GetDBQueries() + assert.Nil(t, queries) + + cfg.DBQueries = repository.New(&pgxpool.Pool{}) + queries = cfg.GetDBQueries() + assert.NotNil(t, queries) + assert.Equal(t, queries, cfg.DBQueries) +} diff --git a/internal/serviceconfig/database/transaction.go b/internal/serviceconfig/database/transaction.go new file mode 100644 index 00000000..9b76b17a --- /dev/null +++ b/internal/serviceconfig/database/transaction.go @@ -0,0 +1,30 @@ +package database + +import ( + "context" + "queryorchestration/internal/database/repository" +) + +func (b *BaseConfig) ExecuteDBTransaction(ctx context.Context, executeQueries func(context.Context, *repository.Queries) error) error { + tx, err := b.DBPool.Begin(ctx) + if err != nil { + return err + } + defer func() { + _ = tx.Rollback(ctx) + }() + + qtx := b.DBQueries.WithTx(tx) + + err = executeQueries(ctx, qtx) + if err != nil { + return err + } + + err = tx.Commit(ctx) + if err != nil { + return err + } + + return nil +} diff --git a/internal/database/connection_test.go b/internal/serviceconfig/database/transaction_test.go similarity index 57% rename from internal/database/connection_test.go rename to internal/serviceconfig/database/transaction_test.go index 0457d453..d0139ea8 100644 --- a/internal/database/connection_test.go +++ b/internal/serviceconfig/database/transaction_test.go @@ -2,11 +2,9 @@ package database_test import ( "context" - "os" - "path" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - "queryorchestration/internal/test" + "queryorchestration/internal/serviceconfig" "testing" "github.com/google/uuid" @@ -14,22 +12,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestDBConn(t *testing.T) { - if testing.Short() { - t.Skip("Skipping long test in short mode") - } - ctx := context.Background() - _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../.."), - }, - }) - defer cleanup() - - pool := database.GetDBPool(ctx) - assert.NotNil(t, pool) -} - func TestExecuteTransaction(t *testing.T) { ctx := context.Background() @@ -37,11 +19,9 @@ func TestExecuteTransaction(t *testing.T) { if err != nil { t.Fatalf("failed to open pgxmock database: %v", err) } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) clientID := database.MustToDBUUID(uuid.New()) clientName := "example_client" @@ -54,7 +34,7 @@ func TestExecuteTransaction(t *testing.T) { ) pool.ExpectCommit() - err = database.ExecuteTransaction(ctx, db, func(ctx context.Context, q *repository.Queries) error { + err = cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error { id, err := q.CreateClient(ctx, "example_client") assert.Nil(t, err) assert.Equal(t, clientID, id) diff --git a/internal/serviceconfig/testlogger.go b/internal/serviceconfig/testlogger.go new file mode 100644 index 00000000..2bbfef85 --- /dev/null +++ b/internal/serviceconfig/testlogger.go @@ -0,0 +1,34 @@ +package serviceconfig + +import ( + "context" + "log/slog" + "testing" +) + +type testLogger struct { + T *testing.T + Logs []map[string]interface{} +} + +func (l *testLogger) Handle(ctx context.Context, r slog.Record) error { + attrs := make(map[string]interface{}) + r.Attrs(func(a slog.Attr) bool { + attrs[a.Key] = a.Value.Any() + return true + }) + l.Logs = append(l.Logs, attrs) + return nil +} + +func (l *testLogger) Enabled(ctx context.Context, level slog.Level) bool { + return true +} + +func (l *testLogger) WithAttrs(attrs []slog.Attr) slog.Handler { + return l +} + +func (l *testLogger) WithGroup(name string) slog.Handler { + return l +} diff --git a/internal/serviceconfig/testlogger_test.go b/internal/serviceconfig/testlogger_test.go new file mode 100644 index 00000000..e3569378 --- /dev/null +++ b/internal/serviceconfig/testlogger_test.go @@ -0,0 +1,25 @@ +package serviceconfig + +import ( + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTestLogger(t *testing.T) { + tl := &testLogger{T: t} + logger := slog.New(tl) + slog.SetDefault(logger) + + slog.Info("test message", "field", "value") + slog.Info("test message", "field", "second") + slog.Warn("test message", "field", "warn") + slog.Error("test message", "field", "error") + + assert.Len(t, tl.Logs, 4) + assert.Equal(t, "value", tl.Logs[0]["field"]) + assert.Equal(t, "second", tl.Logs[1]["field"]) + assert.Equal(t, "warn", tl.Logs[2]["field"]) + assert.Equal(t, "error", tl.Logs[3]["field"]) +} diff --git a/internal/test/api.go b/internal/test/api.go index 84b3a4c4..616ea036 100644 --- a/internal/test/api.go +++ b/internal/test/api.go @@ -26,7 +26,7 @@ func CreateAPIContainer(t *testing.T, ctx context.Context, config *APIContainerC ServiceName: config.ServiceName, Network: config.Network, ExposedPorts: []nat.Port{port}, - WaitForMsg: "Listening on port 8080", + WaitForMsg: "Listening for requests", }) host, err := container.Host(ctx) diff --git a/internal/test/api_test.go b/internal/test/api_test.go index 7d6da905..a408ebd8 100644 --- a/internal/test/api_test.go +++ b/internal/test/api_test.go @@ -4,7 +4,7 @@ import ( "context" "os" "path" - "queryorchestration/internal/database" + "queryorchestration/internal/serviceconfig" "queryorchestration/internal/test" "testing" @@ -19,21 +19,23 @@ func TestCreateAPIContainer(t *testing.T) { ncfg, ncleanup := test.CreateNetwork(t, ctx) defer ncleanup() + + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../..")) + cfg := &serviceconfig.BaseConfig{} dbcfg, dbcleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Network: ncfg, - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../.."), - }, + Network: ncfg, + Cfg: cfg, + RunMigrations: true, }) defer dbcleanup() - cfg := &test.APIContainerConfig{ + acfg := &test.APIContainerConfig{ ServiceName: "queryService", DB: dbcfg.External, Network: ncfg, } - conn, cleanup := test.CreateAPIContainer(t, ctx, cfg) + conn, cleanup := test.CreateAPIContainer(t, ctx, acfg) assert.NotNil(t, conn) assert.NotNil(t, cleanup) diff --git a/internal/test/container.go b/internal/test/container.go index bd145768..405d65ee 100644 --- a/internal/test/container.go +++ b/internal/test/container.go @@ -27,7 +27,7 @@ func createContainer(t *testing.T, ctx context.Context, cfg *containerConfig) (t "DB_HOST": cfg.DB.Host, "DB_NAME": cfg.DB.Name, "DB_PORT": strconv.Itoa(cfg.DB.Port), - "DB_NOSSL": "1", + "DB_NOSSL": "true", } if cfg.Env != nil { for k, v := range cfg.Env { diff --git a/internal/test/container_test.go b/internal/test/container_test.go index d2e5a03e..3446de8e 100644 --- a/internal/test/container_test.go +++ b/internal/test/container_test.go @@ -4,7 +4,7 @@ import ( "context" "os" "path" - "queryorchestration/internal/database" + "queryorchestration/internal/serviceconfig" "testing" "github.com/stretchr/testify/assert" @@ -18,21 +18,23 @@ func TestCreateContainer(t *testing.T) { ncfg, ncleanup := CreateNetwork(t, ctx) defer ncleanup() + + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../..")) + cfg := &serviceconfig.BaseConfig{} dbcfg, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{ - Network: ncfg, - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../.."), - }, + Network: ncfg, + Cfg: cfg, + RunMigrations: true, }) defer dbcleanup() - cfg := &containerConfig{ + ccfg := &containerConfig{ ServiceName: "queryService", DB: dbcfg.External, Network: ncfg, } - container, cleanup := createContainer(t, ctx, cfg) + container, cleanup := createContainer(t, ctx, ccfg) assert.NotNil(t, container) assert.NotNil(t, cleanup) diff --git a/internal/test/database.go b/internal/test/database.go index 4c57960b..9025f677 100644 --- a/internal/test/database.go +++ b/internal/test/database.go @@ -3,11 +3,11 @@ package test import ( "context" "fmt" - "queryorchestration/internal/database" + "queryorchestration/internal/database/migrations" + "queryorchestration/internal/serviceconfig" "testing" "github.com/docker/go-connections/nat" - "github.com/jackc/pgx/v5/pgxpool" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" ) @@ -21,14 +21,14 @@ type ExternalDatabase struct { } type Database struct { - Pool *pgxpool.Pool Container testcontainers.Container External *ExternalDatabase } type CreateDatabaseConfig struct { - Network *testcontainers.DockerNetwork - Migrations *database.MigrationConfig + Network *testcontainers.DockerNetwork + Cfg serviceconfig.ConfigProvider + RunMigrations bool } func CreateDB(t *testing.T, ctx context.Context, cfg *CreateDatabaseConfig) (*Database, func()) { @@ -87,17 +87,24 @@ func CreateDB(t *testing.T, ctx context.Context, cfg *CreateDatabaseConfig) (*Da t.Setenv("DB_HOST", host) t.Setenv("DB_PORT", fmt.Sprint(mappedPort.Int())) t.Setenv("DB_NAME", name) - t.Setenv("DB_NOSSL", "1") + t.Setenv("DB_NOSSL", "true") - var pool *pgxpool.Pool - if cfg.Migrations != nil { - database.RunMigrations(ctx, cfg.Migrations) - - pool = database.GetDBPool(ctx) - - err = pool.Ping(ctx) + if cfg.Cfg != nil { + err = serviceconfig.InitializeConfig(cfg.Cfg) if err != nil { - t.Fatalf("Unable to ping database: %s", err) + t.Fatal(err) + } + + if cfg.RunMigrations { + err := migrations.Run(ctx, cfg.Cfg) + if err != nil { + t.Fatal(err) + } + + err = cfg.Cfg.SetDBPool(ctx) + if err != nil { + t.Fatal(err) + } } } @@ -110,7 +117,6 @@ func CreateDB(t *testing.T, ctx context.Context, cfg *CreateDatabaseConfig) (*Da } return &Database{ - Pool: pool, Container: container, External: external, }, func() { diff --git a/internal/test/database_test.go b/internal/test/database_test.go index a45f6420..0b4e7907 100644 --- a/internal/test/database_test.go +++ b/internal/test/database_test.go @@ -4,7 +4,7 @@ import ( "context" "os" "path" - "queryorchestration/internal/database" + "queryorchestration/internal/serviceconfig" "queryorchestration/internal/test" "testing" @@ -17,9 +17,10 @@ func TestCreateDB(t *testing.T) { } ctx := context.Background() - dbcfg, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{}) + cfg := &serviceconfig.BaseConfig{} + dbcfg, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{Cfg: cfg}) assert.NotNil(t, dbcfg) - assert.Nil(t, dbcfg.Pool) + assert.Nil(t, cfg.DBPool) assert.NotNil(t, cleanup) cleanup() @@ -31,14 +32,15 @@ func TestCreateDBWithMigrations(t *testing.T) { } ctx := context.Background() + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../..")) + cfg := &serviceconfig.BaseConfig{} dbcfg, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../.."), - }, + Cfg: cfg, + RunMigrations: true, }) - assert.NotNil(t, dbcfg) - assert.NotNil(t, dbcfg.Pool) - assert.NotNil(t, cleanup) + defer cleanup() - cleanup() + assert.NotNil(t, dbcfg) + assert.NotNil(t, cfg.DBPool) + assert.NotNil(t, cleanup) } diff --git a/internal/test/queuecontainer_test.go b/internal/test/queuecontainer_test.go index d956cebb..a5a43306 100644 --- a/internal/test/queuecontainer_test.go +++ b/internal/test/queuecontainer_test.go @@ -4,7 +4,7 @@ import ( "context" "os" "path" - "queryorchestration/internal/database" + "queryorchestration/internal/serviceconfig" "testing" "github.com/stretchr/testify/assert" @@ -18,13 +18,16 @@ func TestCreateQueueContainer(t *testing.T) { ncfg, ncleanup := CreateNetwork(t, ctx) defer ncleanup() + + t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../..")) + cfg := &serviceconfig.BaseConfig{} dbcfg, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{ - Migrations: &database.MigrationConfig{ - BasePath: path.Join(os.Getenv("PWD"), "../.."), - }, - Network: ncfg, + Network: ncfg, + Cfg: cfg, + RunMigrations: true, }) defer dbcleanup() + qcfg, qcleanup := CreateQueue(t, ctx, &CreateQueueConfig{ Network: ncfg, }) @@ -36,14 +39,14 @@ func TestCreateQueueContainer(t *testing.T) { Endpoint: qcfg.External.Endpoint, Credentials: qcfg.External.Credentials, } - cfg := &QueueContainerConfig{ + qccfg := &QueueContainerConfig{ ServiceName: "queryRunner", DB: dbcfg.External, Network: ncfg, Queue: extcfg, } - cleanup := CreateQueueContainer(t, ctx, cfg) + cleanup := CreateQueueContainer(t, ctx, qccfg) assert.NotNil(t, cleanup) cleanup()