jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/flakydb (pull request #21)

Browse Source 
Bugfix/flakydb

* init

* fixenvvar

* fixtest
This commit is contained in:
Michael McGuinness
2025-01-14 17:28:26 +00:00
parent 9f9700a773
commit 7da57058d4
17 changed files with 103 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+9 -1
View File
@@ -19,4 +19,12 @@ Using the following project as a baseline: https://github.com/golang-standards/p
To find new packages: `https://search.nixos.org/packages`
For regular usage, please use `task` scripts in order run the most appropriate configuration.
For regular usage, please use `task` scripts in order run the most appropriate configuration.
# Environment Variables
When using environment variables, there is a hiearchy that will be respected. (First will override the next)
1. Within the devbox.json file, a variable added to "env"
2. Within the devbox.json file, a variable exported in "init_hook"
3. A variable added to .env
internal/api/listener_test.go
+4 -2
View File
@@ -2,6 +2,8 @@ package api
import (
"context"
"os"
"path"
"queryorchestration/internal/database"
"queryorchestration/internal/server"
"queryorchestration/internal/test"
@@ -16,7 +18,7 @@ func TestNew(t *testing.T) {
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Migrations: &database.MigrationConfig{
BasePath: "../..",
BasePath: path.Join(os.Getenv("PWD"), "../.."),
},
})
defer cleanup()
@@ -26,7 +28,7 @@ func TestNew(t *testing.T) {
}
cfg := &Config{
GetControllers: getControllers,
BasePath: "../..",
BasePath: path.Join(os.Getenv("PWD"), "../.."),
}
server := New(ctx, cfg)
internal/database/config_test.go
+10 -22
View File
@@ -6,37 +6,36 @@ import (
"github.com/stretchr/testify/assert"
)
func TestMustGetConnectionString(t *testing.T) {
func TestMustGetURIInvalidPort(t *testing.T) {
t.Setenv("DB_USER", "user")
t.Setenv("DB_PASS", "pass")
t.Setenv("DB_HOST", "host")
t.Setenv("DB_PORT", "port")
t.Setenv("DB_NAME", "name")
connStr := mustGetConnectionString()
assert.Equal(t, "postgres://user:pass@host:port/name?", connStr)
assert.Panics(t, func() { mustGetURI() })
}
func TestMustGetConnectionStringNoSSL(t *testing.T) {
func TestMustGetURINoSSL(t *testing.T) {
t.Setenv("DB_USER", "user")
t.Setenv("DB_PASS", "pass")
t.Setenv("DB_HOST", "host")
t.Setenv("DB_PORT", "port")
t.Setenv("DB_PORT", "5432")
t.Setenv("DB_NAME", "name")
t.Setenv("DB_NOSSL", "1")
connStr := mustGetConnectionString()
assert.Equal(t, "postgres://user:pass@host:port/name?sslmode=disable", connStr)
connStr := mustGetURI()
assert.Equal(t, "postgres://user:pass@host:5432/name?sslmode=disable", connStr.String())
}
func TestMustGetConnectionStringNoEnv(t *testing.T) {
func TestMustGetURINoEnv(t *testing.T) {
t.Setenv("DB_USER", "")
t.Setenv("DB_PASS", "")
t.Setenv("DB_HOST", "")
t.Setenv("DB_PORT", "")
t.Setenv("DB_NAME", "")
assert.Panics(t, func() { mustGetConnectionString() })
assert.Panics(t, func() { mustGetURI() })
}
func TestGetPoolConfig(t *testing.T) {
@@ -48,18 +47,7 @@ func TestGetPoolConfig(t *testing.T) {
config, err := getPoolConfig()
assert.Nil(t, err)
assert.Equal(t, config.ConnString(), mustGetConnectionString())
}
func TestGetPoolConfigInvalidPort(t *testing.T) {
t.Setenv("DB_USER", "user")
t.Setenv("DB_PASS", "pass")
t.Setenv("DB_HOST", "host")
t.Setenv("DB_PORT", "invalid_port")
t.Setenv("DB_NAME", "name")
_, err := getPoolConfig()
assert.EqualError(t, err, "cannot parse `postgres://user:xxxxxx@host:invalid_port/name?`: failed to parse as URL (invalid port \":invalid_port\" after host)")
assert.Equal(t, config.ConnString(), mustGetURI().String())
}
func TestMustGetPoolConfig(t *testing.T) {
@@ -70,7 +58,7 @@ func TestMustGetPoolConfig(t *testing.T) {
t.Setenv("DB_NAME", "name")
config := mustGetPoolConfig()
assert.Equal(t, config.ConnString(), mustGetConnectionString())
assert.Equal(t, config.ConnString(), mustGetURI().String())
}
func TestMustGetPoolConfigInvalidPort(t *testing.T) {
internal/database/connection.go
+21 -8
View File
@@ -4,8 +4,10 @@ import (
"context"
"fmt"
"log"
"net/url"
"queryorchestration/internal/env"
"github.com/docker/go-connections/nat"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
@@ -14,7 +16,7 @@ type config struct {
user string
password string
host string
port string
port nat.Port
name string
disableSSL bool
}
@@ -24,6 +26,10 @@ func mustGetConfig() *config {
pass := env.GetPanic("DB_PASS")
host := env.GetPanic("DB_HOST")
port := env.GetPanic("DB_PORT")
natPort, err := nat.NewPort("tcp", port)
if err != nil {
log.Panicf("Failed to create port: %v", err)
}
name := env.GetPanic("DB_NAME")
disableSSL, _ := env.Get("DB_NOSSL")
@@ -32,13 +38,13 @@ func mustGetConfig() *config {
user: user,
password: pass,
host: host,
port: port,
port: natPort,
name: name,
disableSSL: disableSSL == "1",
}
}
func mustGetConnectionString() string {
func mustGetURI() *url.URL {
driver := "postgres"
conf := mustGetConfig()
opts := ""
@@ -47,13 +53,20 @@ func mustGetConnectionString() string {
opts += "sslmode=disable"
}
return fmt.Sprintf("%s://%s:%s@%s:%s/%s?%s", driver, conf.user, conf.password, conf.host, conf.port, conf.name, opts)
connStr := fmt.Sprintf("%s://%s:%s@%s:%d/%s?%s", driver, conf.user, conf.password, conf.host, conf.port.Int(), conf.name, opts)
uri, err := url.Parse(connStr)
if err != nil {
log.Panicf("Unable to parse URI: %s", err)
}
return uri
}
func getPoolConfig() (*pgxpool.Config, error) {
connStr := mustGetConnectionString()
connStr := mustGetURI()
config, err := pgxpool.ParseConfig(connStr)
config, err := pgxpool.ParseConfig(connStr.String())
if err != nil {
return nil, err
}
@@ -82,9 +95,9 @@ func GetDBPool(ctx context.Context) *pgxpool.Pool {
}
func GetDBConn(ctx context.Context) *pgx.Conn {
config := mustGetConnectionString()
config := mustGetURI()
conn, err := pgx.Connect(ctx, config)
conn, err := pgx.Connect(ctx, config.String())
if err != nil {
log.Panicf("Unable to connect to database: %v\n", err)
}
internal/database/connection_test.go
+3 -1
View File
@@ -2,6 +2,8 @@ package database_test
import (
"context"
"os"
"path"
"queryorchestration/internal/database"
"queryorchestration/internal/test"
"testing"
@@ -13,7 +15,7 @@ func TestDBConn(t *testing.T) {
ctx := context.Background()
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Migrations: &database.MigrationConfig{
BasePath: "../..",
BasePath: path.Join(os.Getenv("PWD"), "../.."),
},
})
defer cleanup()
internal/database/migrations.go
+15 -35
View File
@@ -9,31 +9,33 @@ import (
"path"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database/postgres"
_ "github.com/golang-migrate/migrate/v4/database/postgres"
_ "github.com/golang-migrate/migrate/v4/source/file"
"github.com/jackc/pgx/v5"
_ "github.com/lib/pq"
)
func createDB(ctx context.Context) {
func createDB() {
driver := "postgres"
opts := ""
conf := mustGetConfig()
opts := ""
if conf.disableSSL {
opts += "sslmode=disable"
}
connStr := fmt.Sprintf("%s://%s:%s@%s:%s?%s", driver, conf.user, conf.password, conf.host, conf.port, opts)
conn, err := pgx.Connect(ctx, connStr)
connStr := fmt.Sprintf("%s://%s:%s@%s:%d/?%s", driver, conf.user, conf.password, conf.host, conf.port.Int(), opts)
db, err := sql.Open("postgres", connStr)
if err != nil {
log.Panicf("Error opening database: %v", err)
}
err = conn.Ping(ctx)
err = db.Ping()
if err != nil {
log.Panicf("Error pinging database: %v", err)
}
rs, err := conn.Query(ctx, fmt.Sprintf("SELECT 'CREATE DATABASE %s' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '%s')", conf.name, conf.name))
rs, err := db.Query(fmt.Sprintf("SELECT 'CREATE DATABASE %s' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '%s')", conf.name, conf.name))
if err != nil {
log.Panicf("Error creating database: %v", err)
}
@@ -46,45 +48,23 @@ func createDB(ctx context.Context) {
}
type MigrationConfig struct {
BasePath string
ConnectionString string
BasePath string
}
func RunMigrations(ctx context.Context, config *MigrationConfig) {
// createDB(ctx) - Add back only if necessary
createDB()
connStr := config.ConnectionString
if connStr == "" {
connStr = mustGetConnectionString()
}
connStr := mustGetURI()
basePath := config.BasePath
if basePath == "" {
basePath = os.Getenv("PWD")
}
driver := "postgres"
db, err := sql.Open(driver, connStr)
if err != nil {
log.Panicf("failed to open database: %v", err)
}
defer db.Close()
if err = db.Ping(); err != nil {
log.Panic("failed to connect to database:", err)
}
conn, err := postgres.WithInstance(db, &postgres.Config{})
if err != nil {
log.Panic("failed to create migrations: ", err)
}
migPath := "file://" + path.Join(basePath, "database/migrations")
m, err := migrate.NewWithDatabaseInstance(
m, err := migrate.New(
migPath,
driver,
conn,
connStr.String(),
)
if err != nil {
log.Panicf("failed to create migrate instance: %v", err)
internal/database/migrations_test.go
-2
View File
@@ -19,8 +19,6 @@ func TestRunMigrations(t *testing.T) {
})
defer cleanup()
t.Setenv("DB_NOSSL", "1")
database.RunMigrations(ctx, &database.MigrationConfig{
BasePath: path.Join(os.Getenv("PWD"), "../.."),
})
internal/database/migrationsprivate_test.go
+1 -3
View File
@@ -1,13 +1,11 @@
package database
import (
"context"
"testing"
)
func TestCreateDB(t *testing.T) {
t.Skip("Must test appropriately")
ctx := context.Background()
createDB(ctx)
createDB()
}
internal/database/repository/collector_test.go
+3 -1
View File
@@ -2,6 +2,8 @@ package repository_test
import (
"context"
"os"
"path"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/test"
@@ -15,7 +17,7 @@ func TestCollector(t *testing.T) {
ctx := context.Background()
db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Migrations: &database.MigrationConfig{
BasePath: "../../..",
BasePath: path.Join(os.Getenv("PWD"), "../../.."),
}})
defer cleanup()
internal/database/repository/query_test.go
+3 -1
View File
@@ -3,6 +3,8 @@ package repository_test
import (
"context"
"log"
"os"
"path"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/test"
@@ -17,7 +19,7 @@ func TestQueries(t *testing.T) {
db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Migrations: &database.MigrationConfig{
BasePath: "../../..",
BasePath: path.Join(os.Getenv("PWD"), "../../.."),
}})
defer cleanup()
internal/database/repository/result_test.go
+3 -1
View File
@@ -2,6 +2,8 @@ package repository_test
import (
"context"
"os"
"path"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/test"
@@ -17,7 +19,7 @@ func TestResults(t *testing.T) {
db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Migrations: &database.MigrationConfig{
BasePath: "../../..",
BasePath: path.Join(os.Getenv("PWD"), "../../.."),
}})
defer cleanup()
internal/queue/listener_test.go
+3 -1
View File
@@ -2,6 +2,8 @@ package queue_test
import (
"context"
"os"
"path"
"queryorchestration/internal/queue"
"queryorchestration/internal/server"
"queryorchestration/internal/test"
@@ -29,7 +31,7 @@ func TestNew(t *testing.T) {
queue.NewServer(ctx, &queue.ListenerConfig{
Controller: controller,
BasePath: "../..",
BasePath: path.Join(os.Getenv("PWD"), "../.."),
})
}
internal/server/server_test.go
+3 -1
View File
@@ -2,6 +2,8 @@ package server_test
import (
"context"
"os"
"path"
"queryorchestration/internal/server"
"queryorchestration/internal/test"
"testing"
@@ -18,7 +20,7 @@ func TestNew(t *testing.T) {
defer cleanup()
newCfg := &server.NewConfig{
BasePath: "../..",
BasePath: path.Join(os.Getenv("PWD"), "../.."),
}
cfg := server.New(ctx, newCfg)
internal/test/api_test.go
+3 -1
View File
@@ -2,6 +2,8 @@ package test_test
import (
"context"
"os"
"path"
"queryorchestration/internal/database"
"queryorchestration/internal/test"
"testing"
@@ -17,7 +19,7 @@ func TestCreateAPIContainer(t *testing.T) {
dbcfg, dbcleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Network: ncfg,
Migrations: &database.MigrationConfig{
BasePath: "../..",
BasePath: path.Join(os.Getenv("PWD"), "../.."),
},
})
defer dbcleanup()
internal/test/container_test.go
+3 -1
View File
@@ -2,6 +2,8 @@ package test
import (
"context"
"os"
"path"
"queryorchestration/internal/database"
"testing"
@@ -16,7 +18,7 @@ func TestCreateContainer(t *testing.T) {
dbcfg, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Network: ncfg,
Migrations: &database.MigrationConfig{
BasePath: "../..",
BasePath: path.Join(os.Getenv("PWD"), "../.."),
},
})
defer dbcleanup()
internal/test/database.go
+2 -1
View File
@@ -52,7 +52,8 @@ func CreateDB(t *testing.T, ctx context.Context, cfg *CreateDatabaseConfig) (*Da
"POSTGRES_PASSWORD": pass,
},
ExposedPorts: []string{port.Port()},
WaitingFor: wait.ForListeningPort(port),
// WaitingFor: wait.ForLog("database system is ready to accept connections"),
WaitingFor: wait.ForExposedPort(),
}
if cfg.Network != nil {
internal/test/database_test.go
+17
View File
@@ -2,6 +2,9 @@ package test_test
import (
"context"
"os"
"path"
"queryorchestration/internal/database"
"queryorchestration/internal/test"
"testing"
@@ -19,3 +22,17 @@ func TestCreateDB(t *testing.T) {
cleanup()
}
func TestCreateDBWithMigrations(t *testing.T) {
ctx := context.Background()
dbcfg, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Migrations: &database.MigrationConfig{
BasePath: path.Join(os.Getenv("PWD"), "../.."),
},
})
assert.NotNil(t, dbcfg)
assert.NotNil(t, cleanup)
cleanup()
}