jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/postprocessing (pull request #114)

Browse Source 
Feature/postprocessing

* tests

* passtest

* fixshorttests

* mosttests

* improvingbasedockerfile

* testspeeds

* testing

* host

* canparallel

* clean

* passfullsuite

* singlepagemax

* test

* findfeatures

* findstables

* tbls

* tablestoo

* tablestoo

* lateraltests

* tableloc

* cleanup

* inlinetable

* childids

* cleanup

* tests
This commit is contained in:
Michael McGuinness
2025-04-22 14:40:16 +00:00
parent edc50c7510
commit fee71e7740
404 changed files with 211048 additions and 2820 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/serviceconfig/database/config.go
+5
View File
@@ -25,6 +25,7 @@ type DBConfig struct {
type ConfigProvider interface {
GetDBBaseURI() string
GetDBURI() string
GetDBAdminDBURI() string
GetDBOpts() map[string]string
GetDBOptsString() string
GetDBHost() string
@@ -82,6 +83,10 @@ func (b *DBConfig) GetDBBaseURI() string {
return fmt.Sprintf("%s://%s:%s@%s:%d/", b.GetDBDriver(), b.DBUser, b.DBSecret, b.DBHost, b.DBPort)
}
func (b *DBConfig) GetDBAdminDBURI() string {
return fmt.Sprintf("%s%s?%s", b.GetDBBaseURI(), b.GetDBDriver(), b.GetDBOptsString())
}
func (b *DBConfig) GetDBURI() string {
return fmt.Sprintf("%s%s?%s", b.GetDBBaseURI(), b.DBName, b.GetDBOptsString())
}
internal/serviceconfig/database/config_test.go
+16
View File
@@ -67,6 +67,22 @@ func TestGetDBURI(t *testing.T) {
assert.Equal(t, "postgres://user:pass@host:123/name?sslmode=disable", uri)
}
func TestGetDBAdminDBURI(t *testing.T) {
cfg := database.DBConfig{}
uri := cfg.GetDBURI()
assert.Equal(t, "postgres://:@:0/?", uri)
cfg.DBHost = "host"
cfg.DBPort = 123
cfg.DBUser = "user"
cfg.DBSecret = "pass"
cfg.DBName = "name"
cfg.DBNoSSL = true
uri = cfg.GetDBAdminDBURI()
assert.Equal(t, "postgres://user:pass@host:123/postgres?sslmode=disable", uri)
}
func TestGetDBHost(t *testing.T) {
cfg := database.DBConfig{}
internal/serviceconfig/database/pool_test.go
+2 -2
View File
@@ -22,10 +22,10 @@ func TestSetDBPool(t *testing.T) {
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
err := cfg.SetDBPool(ctx)
require.NoError(t, err)
internal/serviceconfig/objectstore/bucketkey.go
+4 -5
View File
@@ -24,10 +24,9 @@ type BucketKey struct {
}
const (
Import BucketLocation = "import"
TextTextract BucketLocation = "text/textract"
TextOut BucketLocation = "text/out"
Export BucketLocation = "export"
Import BucketLocation = "import"
Text BucketLocation = "text"
Export BucketLocation = "export"
)
var PartExclusions = []BucketLocation{
@@ -133,7 +132,7 @@ func (c *BucketKey) parseClientID(name string) error {
}
func (c *BucketKey) parseLocation(name string) error {
c.Location = BucketLocation(strings.ReplaceAll(name, "-", "/"))
pattern := fmt.Sprintf(`^(%s|%s|%s|%s)$`, Import, TextTextract, TextOut, Export)
pattern := fmt.Sprintf(`^(%s|%s|%s)$`, Import, Text, Export)
reg := regexp.MustCompile(pattern)
matches := reg.FindStringSubmatch(string(c.Location))
if len(matches) != 2 {
internal/serviceconfig/objectstore/bucketkey_test.go
+8 -31
View File
@@ -19,20 +19,16 @@ func TestGetBucketKey(t *testing.T) {
assert.Equal(t, "AAA//20250331/0/2025-03-31T040816Z_AAA__b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = Import
assert.Equal(t, "AAA/import/20250331/0/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = TextTextract
assert.Equal(t, "AAA/text/textract/20250331/0/2025-03-31T040816Z_AAA_text-textract_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = TextOut
assert.Equal(t, "AAA/text/out/20250331/0/2025-03-31T040816Z_AAA_text-out_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = Text
assert.Equal(t, "AAA/text/20250331/0/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = Export
assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
part := uint16(3)
key.Part = &part
key.Location = Import
assert.Equal(t, "AAA/import/20250331/3/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = TextTextract
assert.Equal(t, "AAA/text/textract/20250331/3/2025-03-31T040816Z_AAA_text-textract_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = TextOut
assert.Equal(t, "AAA/text/out/20250331/3/2025-03-31T040816Z_AAA_text-out_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = Text
assert.Equal(t, "AAA/text/20250331/3/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = Export
assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
filetype := "csv"
@@ -67,24 +63,15 @@ func TestParseBucketKey(t *testing.T) {
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
}, key)
key, err = ParseBucketKey("AAA/text/textract/20250331/2025-03-31T040816Z_AAA_text-textract_b745910f-f529-43c5-87e1-25629d46ef40")
key, err = ParseBucketKey("AAA/text/20250331/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
ClientID: "AAA",
Location: TextTextract,
Location: Text,
EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
CreatedAt: time.Date(2025, 2, 31, 0, 0, 0, 0, time.UTC),
}, key)
key, err = ParseBucketKey("AAA/text/out/20250331/2025-03-31T040816Z_AAA_text-out_b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
ClientID: "AAA",
Location: TextOut,
EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
}, key)
key, err = ParseBucketKey("AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
@@ -105,21 +92,11 @@ func TestParseBucketKey(t *testing.T) {
Part: &part,
}, key)
key, err = ParseBucketKey("AAA/text/textract/20250331/3/2025-03-31T040816Z_AAA_text-textract_b745910f-f529-43c5-87e1-25629d46ef40")
key, err = ParseBucketKey("AAA/text/20250331/3/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
ClientID: "AAA",
Location: TextTextract,
EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
Part: &part,
}, key)
key, err = ParseBucketKey("AAA/text/out/20250331/3/2025-03-31T040816Z_AAA_text-out_b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
ClientID: "AAA",
Location: TextOut,
Location: Text,
EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
Part: &part,
internal/serviceconfig/objectstore/config_test.go
+2 -1
View File
@@ -94,7 +94,8 @@ func TestCalculateETag(t *testing.T) {
cfg := &StoreConfig{}
test.SetCfgProvider(t, cfg)
acfg, clean := test.CreateAWSContainer(t, ctx, cfg, &test.CreateAWSConfig{})
net := test.DepNetwork.Get(t, ctx)
acfg, clean := test.CreateAWSContainer(t, ctx, cfg, net)
defer clean()
test.SetStoreClient(t, ctx, cfg, acfg.ExternalEndpoint)
test.CreateBucket(t, ctx, cfg, test.BucketName)
internal/serviceconfig/queue/documenttextprocess/config.go
-15
View File
@@ -1,15 +0,0 @@
package documenttextprocess
const EnvName = "DOCUMENT_TEXT_PROCESS_URL"
type DocTextProcessConfig struct {
DocumentTextProcessURL string `env:"DOCUMENT_TEXT_PROCESS_URL,required,notEmpty"`
}
func (c *DocTextProcessConfig) GetDocumentTextProcessURL() string {
return c.DocumentTextProcessURL
}
type ConfigProvider interface {
GetDocumentTextProcessURL() string
}
internal/serviceconfig/queue/documenttextprocess/config_test.go
-21
View File
@@ -1,21 +0,0 @@
package documenttextprocess_test
import (
"testing"
"queryorchestration/internal/serviceconfig/queue/documenttextprocess"
"github.com/stretchr/testify/assert"
)
func TestGetDocumentTextURL(t *testing.T) {
cfg := documenttextprocess.DocTextProcessConfig{}
name := cfg.GetDocumentTextProcessURL()
assert.Equal(t, "", name)
cfg.DocumentTextProcessURL = "name"
name = cfg.GetDocumentTextProcessURL()
assert.Equal(t, "name", name)
assert.Equal(t, cfg.DocumentTextProcessURL, name)
}