jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/postprocessing (pull request #114)

Browse Source 
Feature/postprocessing

* tests

* passtest

* fixshorttests

* mosttests

* improvingbasedockerfile

* testspeeds

* testing

* host

* canparallel

* clean

* passfullsuite

* singlepagemax

* test

* findfeatures

* findstables

* tbls

* tablestoo

* tablestoo

* lateraltests

* tableloc

* cleanup

* inlinetable

* childids

* cleanup

* tests
This commit is contained in:
Michael McGuinness
2025-04-22 14:40:16 +00:00
parent edc50c7510
commit fee71e7740
404 changed files with 211048 additions and 2820 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/repository/clean_test.go
+3 -3
View File
@@ -13,17 +13,17 @@ import (
)
func TestClean(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
internal/database/repository/client_test.go
+3 -3
View File
@@ -13,17 +13,17 @@ import (
)
func TestClient(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
internal/database/repository/collector_test.go
+3 -3
View File
@@ -15,17 +15,17 @@ import (
)
func TestCollector(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
internal/database/repository/document_test.go
+3 -3
View File
@@ -13,17 +13,17 @@ import (
)
func TestDocument(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
internal/database/repository/models.go
+4 -18
View File
@@ -19,7 +19,6 @@ const (
CleanfailtypeInvalidRead Cleanfailtype = "invalid_read"
CleanfailtypeInvalidReadPages Cleanfailtype = "invalid_read_pages"
CleanfailtypeZeroPageCount Cleanfailtype = "zero_page_count"
CleanfailtypeLargePageCount Cleanfailtype = "large_page_count"
CleanfailtypeLargeFile Cleanfailtype = "large_file"
CleanfailtypeSmallDimensions Cleanfailtype = "small_dimensions"
CleanfailtypeLargeDimensions Cleanfailtype = "large_dimensions"
@@ -68,7 +67,6 @@ func (e Cleanfailtype) Valid() bool {
CleanfailtypeInvalidRead,
CleanfailtypeInvalidReadPages,
CleanfailtypeZeroPageCount,
CleanfailtypeLargePageCount,
CleanfailtypeLargeFile,
CleanfailtypeSmallDimensions,
CleanfailtypeLargeDimensions,
@@ -290,9 +288,6 @@ type Currenttextentry struct {
Key string `db:"key"`
Hash string `db:"hash"`
Cleanid uuid.UUID `db:"cleanid"`
Triggerid uuid.UUID `db:"triggerid"`
Textractjobid *string `db:"textractjobid"`
Triggerversion int64 `db:"triggerversion"`
Extractionversion int64 `db:"extractionversion"`
}
@@ -327,6 +322,7 @@ type Documententry struct {
type Documenttextextraction struct {
ID uuid.UUID `db:"id"`
Cleanid uuid.UUID `db:"cleanid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Hash string `db:"hash"`
@@ -335,19 +331,9 @@ type Documenttextextraction struct {
}
type Documenttextextractionentry struct {
ID uuid.UUID `db:"id"`
Textid uuid.UUID `db:"textid"`
Triggerid uuid.UUID `db:"triggerid"`
Version int64 `db:"version"`
}
type Documenttexttriggerextraction struct {
ID uuid.UUID `db:"id"`
Cleanid uuid.UUID `db:"cleanid"`
Version int64 `db:"version"`
Createdat pgtype.Timestamp `db:"createdat"`
Part uint16 `db:"part"`
Textractjobid *string `db:"textractjobid"`
ID uuid.UUID `db:"id"`
Textid uuid.UUID `db:"textid"`
Version int64 `db:"version"`
}
type Fullactivecollector struct {
internal/database/repository/query_test.go
+12 -12
View File
@@ -14,17 +14,17 @@ import (
)
func TestQueries(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
@@ -226,17 +226,17 @@ func TestQueries(t *testing.T) {
}
func TestQueryDependencyTree(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
@@ -447,17 +447,17 @@ func TestQueryDependencyTree(t *testing.T) {
}
func TestQueriesList(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
@@ -505,17 +505,17 @@ func TestQueriesList(t *testing.T) {
}
func TestListQueryClients(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
internal/database/repository/result_test.go
+31 -71
View File
@@ -16,17 +16,17 @@ import (
)
func TestResults(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
@@ -97,9 +97,11 @@ func TestResults(t *testing.T) {
require.NoError(t, err)
assert.False(t, issynced)
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Cleanid: cleanid,
Version: 123,
Bucket: "hi",
Key: "hello",
Hash: "example",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
@@ -107,21 +109,9 @@ func TestResults(t *testing.T) {
},
})
require.NoError(t, err)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
Hash: "example",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Textid: textId,
Triggerid: triggerId,
Version: 1,
Textid: textId,
})
require.NoError(t, err)
@@ -167,17 +157,17 @@ func TestResults(t *testing.T) {
}
func TestResultValues(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
@@ -274,9 +264,11 @@ func TestResultValues(t *testing.T) {
Version: 1,
})
require.NoError(t, err)
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Cleanid: cleanid,
Version: 123,
Bucket: "hi",
Key: "hello",
Hash: "example",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
@@ -284,21 +276,9 @@ func TestResultValues(t *testing.T) {
},
})
require.NoError(t, err)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
Hash: "example",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Textid: textId,
Triggerid: triggerId,
Version: 1,
Textid: textId,
})
require.NoError(t, err)
@@ -393,17 +373,17 @@ func TestResultValues(t *testing.T) {
}
func TestUnsyncedNoDepsQueries(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
@@ -498,9 +478,10 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
require.NoError(t, err)
assert.Len(t, qs, 0)
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Cleanid: cleanid,
Version: 123,
Bucket: "hi",
Key: "hello",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
@@ -508,20 +489,9 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
},
})
require.NoError(t, err)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Textid: textId,
Triggerid: triggerId,
Version: 1,
Textid: textId,
})
require.NoError(t, err)
@@ -588,9 +558,10 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
require.NoError(t, err)
assert.Len(t, qs, 0)
triggerTwoId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
textTwoId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Cleanid: cleantwoid,
Version: 123,
Bucket: "hi",
Key: "hello",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
@@ -598,20 +569,9 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
},
})
require.NoError(t, err)
textTwoId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Textid: textTwoId,
Triggerid: triggerTwoId,
Version: 1,
Textid: textTwoId,
})
require.NoError(t, err)
internal/database/repository/sync_test.go
+15 -45
View File
@@ -15,17 +15,17 @@ import (
)
func TestListClientDocumentIDs(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
@@ -120,17 +120,17 @@ func TestListClientDocumentIDs(t *testing.T) {
}
func TestClientSync(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
@@ -285,31 +285,21 @@ func TestClientSync(t *testing.T) {
Fields: []byte(`{"first_key": null}`),
}, docExternal)
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Cleanid: cleanId,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
Hash: "example",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Textid: textId,
Triggerid: triggerId,
Version: 1,
Textid: textId,
})
require.NoError(t, err)
@@ -467,16 +457,6 @@ func TestClientSync(t *testing.T) {
})
t.Run("update text entry", func(t *testing.T) {
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanId,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
@@ -485,12 +465,12 @@ func TestClientSync(t *testing.T) {
Time: time.Now().UTC(),
Valid: true,
},
Cleanid: cleanId,
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Textid: textId,
Triggerid: triggerId,
Version: 1,
Textid: textId,
})
require.NoError(t, err)
@@ -594,16 +574,6 @@ func TestClientSync(t *testing.T) {
Fields: []byte(`{"first_key": null}`),
}, docExternal)
triggerThreeId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanthreeid,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
textThreeId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
@@ -612,12 +582,12 @@ func TestClientSync(t *testing.T) {
Time: time.Now().UTC(),
Valid: true,
},
Cleanid: cleanthreeid,
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Textid: textThreeId,
Triggerid: triggerThreeId,
Version: 1,
Textid: textThreeId,
})
require.NoError(t, err)
internal/database/repository/text.sql.go
+29 -130
View File
@@ -14,12 +14,13 @@ import (
const addDocumentText = `-- name: AddDocumentText :one
INSERT INTO documentTextExtractions
(bucket, key, hash, createdAt, part)
VALUES ($1, $2, $3, $4, $5)
(cleanId, bucket, key, hash, createdAt, part)
VALUES ($1, $2, $3, $4, $5, $6)
returning id
`
type AddDocumentTextParams struct {
Cleanid uuid.UUID `db:"cleanid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Hash string `db:"hash"`
@@ -30,11 +31,12 @@ type AddDocumentTextParams struct {
// AddDocumentText
//
// INSERT INTO documentTextExtractions
// (bucket, key, hash, createdAt, part)
// VALUES ($1, $2, $3, $4, $5)
// (cleanId, bucket, key, hash, createdAt, part)
// VALUES ($1, $2, $3, $4, $5, $6)
// returning id
func (q *Queries) AddDocumentText(ctx context.Context, arg *AddDocumentTextParams) (uuid.UUID, error) {
row := q.db.QueryRow(ctx, addDocumentText,
arg.Cleanid,
arg.Bucket,
arg.Key,
arg.Hash,
@@ -48,76 +50,22 @@ func (q *Queries) AddDocumentText(ctx context.Context, arg *AddDocumentTextParam
const addDocumentTextEntry = `-- name: AddDocumentTextEntry :exec
INSERT INTO documentTextExtractionEntries
(textId, triggerId, version)
VALUES ($1, $2, $3)
(textId, version)
VALUES ($1, $2)
`
type AddDocumentTextEntryParams struct {
Textid uuid.UUID `db:"textid"`
Triggerid uuid.UUID `db:"triggerid"`
Version int64 `db:"version"`
Textid uuid.UUID `db:"textid"`
Version int64 `db:"version"`
}
// AddDocumentTextEntry
//
// INSERT INTO documentTextExtractionEntries
// (textId, triggerId, version)
// VALUES ($1, $2, $3)
// (textId, version)
// VALUES ($1, $2)
func (q *Queries) AddDocumentTextEntry(ctx context.Context, arg *AddDocumentTextEntryParams) error {
_, err := q.db.Exec(ctx, addDocumentTextEntry, arg.Textid, arg.Triggerid, arg.Version)
return err
}
const addDocumentTextTrigger = `-- name: AddDocumentTextTrigger :one
INSERT INTO documentTextTriggerExtractions
(cleanId, version, createdAt, part) VALUES
($1, $2, $3, $4)
returning id
`
type AddDocumentTextTriggerParams struct {
Cleanid uuid.UUID `db:"cleanid"`
Version int64 `db:"version"`
Createdat pgtype.Timestamp `db:"createdat"`
Part uint16 `db:"part"`
}
// AddDocumentTextTrigger
//
// INSERT INTO documentTextTriggerExtractions
// (cleanId, version, createdAt, part) VALUES
// ($1, $2, $3, $4)
// returning id
func (q *Queries) AddDocumentTextTrigger(ctx context.Context, arg *AddDocumentTextTriggerParams) (uuid.UUID, error) {
row := q.db.QueryRow(ctx, addDocumentTextTrigger,
arg.Cleanid,
arg.Version,
arg.Createdat,
arg.Part,
)
var id uuid.UUID
err := row.Scan(&id)
return id, err
}
const addDocumentTextTriggerJobId = `-- name: AddDocumentTextTriggerJobId :exec
UPDATE documentTextTriggerExtractions
SET textractJobId = $1
WHERE id = $2
`
type AddDocumentTextTriggerJobIdParams struct {
Textractjobid *string `db:"textractjobid"`
ID uuid.UUID `db:"id"`
}
// AddDocumentTextTriggerJobId
//
// UPDATE documentTextTriggerExtractions
// SET textractJobId = $1
// WHERE id = $2
func (q *Queries) AddDocumentTextTriggerJobId(ctx context.Context, arg *AddDocumentTextTriggerJobIdParams) error {
_, err := q.db.Exec(ctx, addDocumentTextTriggerJobId, arg.Textractjobid, arg.ID)
_, err := q.db.Exec(ctx, addDocumentTextEntry, arg.Textid, arg.Version)
return err
}
@@ -140,51 +88,15 @@ func (q *Queries) GetDocumentTextExtractionByHash(ctx context.Context, arg *GetD
return id, err
}
const getDocumentTextTrigger = `-- name: GetDocumentTextTrigger :one
SELECT dt.id, dt.cleanId, dt.version, dt.textractJobId, dc.documentId
from documentTextTriggerExtractions as dt
join documentCleans as dc on dt.cleanId = dc.id
where dt.id = $1
`
type GetDocumentTextTriggerRow struct {
ID uuid.UUID `db:"id"`
Cleanid uuid.UUID `db:"cleanid"`
Version int64 `db:"version"`
Textractjobid *string `db:"textractjobid"`
Documentid uuid.UUID `db:"documentid"`
}
// GetDocumentTextTrigger
//
// SELECT dt.id, dt.cleanId, dt.version, dt.textractJobId, dc.documentId
// from documentTextTriggerExtractions as dt
// join documentCleans as dc on dt.cleanId = dc.id
// where dt.id = $1
func (q *Queries) GetDocumentTextTrigger(ctx context.Context, triggerid uuid.UUID) (*GetDocumentTextTriggerRow, error) {
row := q.db.QueryRow(ctx, getDocumentTextTrigger, triggerid)
var i GetDocumentTextTriggerRow
err := row.Scan(
&i.ID,
&i.Cleanid,
&i.Version,
&i.Textractjobid,
&i.Documentid,
)
return &i, err
}
const getTextEntryByDocId = `-- name: GetTextEntryByDocId :one
SELECT id, documentId, bucket, key, hash, cleanId,
triggerId, textractJobId, triggerVersion, extractionVersion
SELECT id, documentId, bucket, key, hash, cleanId, extractionVersion
FROM currentTextEntries
WHERE documentId = $1
`
// GetTextEntryByDocId
//
// SELECT id, documentId, bucket, key, hash, cleanId,
// triggerId, textractJobId, triggerVersion, extractionVersion
// SELECT id, documentId, bucket, key, hash, cleanId, extractionVersion
// FROM currentTextEntries
// WHERE documentId = $1
func (q *Queries) GetTextEntryByDocId(ctx context.Context, documentid uuid.UUID) (*Currenttextentry, error) {
@@ -197,9 +109,6 @@ func (q *Queries) GetTextEntryByDocId(ctx context.Context, documentid uuid.UUID)
&i.Key,
&i.Hash,
&i.Cleanid,
&i.Triggerid,
&i.Textractjobid,
&i.Triggerversion,
&i.Extractionversion,
)
return &i, err
@@ -210,17 +119,12 @@ WITH client as (
select id from clients where id = $1
),
parts as (
SELECT extractions.part
SELECT extract.part
FROM client as c
JOIN documents as docs on docs.clientId = c.id
JOIN documentCleans as clean on docs.id = clean.documentId
JOIN documentTextTriggerExtractions as triggers
on triggers.cleanId = clean.id
JOIN documentTextExtractionEntries as textEntries
on textEntries.triggerId = triggers.id
JOIN documentTextExtractions as extractions
on extractions.id = textEntries.textId
WHERE date(extractions.createdAt) = date($2)
JOIN documentTextExtractions extract ON extract.cleanId = clean.id
WHERE date(extract.createdAt) = date($2)
),
max_part as (
SELECT COALESCE(max(part), 0) as max_part_num FROM parts
@@ -249,17 +153,12 @@ type GetTextOutCurrentPartRow struct {
// select id from clients where id = $1
// ),
// parts as (
// SELECT extractions.part
// SELECT extract.part
// FROM client as c
// JOIN documents as docs on docs.clientId = c.id
// JOIN documentCleans as clean on docs.id = clean.documentId
// JOIN documentTextTriggerExtractions as triggers
// on triggers.cleanId = clean.id
// JOIN documentTextExtractionEntries as textEntries
// on textEntries.triggerId = triggers.id
// JOIN documentTextExtractions as extractions
// on extractions.id = textEntries.textId
// WHERE date(extractions.createdAt) = date($2)
// JOIN documentTextExtractions extract ON extract.cleanId = clean.id
// WHERE date(extract.createdAt) = date($2)
// ),
// max_part as (
// SELECT COALESCE(max(part), 0) as max_part_num FROM parts
@@ -282,13 +181,13 @@ WITH client as (
select id from clients where id = $1
),
parts as (
SELECT triggers.part
SELECT extract.part
FROM client as c
JOIN documents as docs on docs.clientId = c.id
JOIN documentCleans as clean on docs.id = clean.documentId
JOIN documentTextTriggerExtractions as triggers
on triggers.cleanId = clean.id
WHERE date(triggers.createdAt) = date($2)
JOIN documentTextExtractions as extract
on extract.cleanId = clean.id
WHERE date(extract.createdAt) = date($2)
),
max_part as (
SELECT COALESCE(max(part), 0) as max_part_num FROM parts
@@ -317,13 +216,13 @@ type GetTextractOutputCurrentPartRow struct {
// select id from clients where id = $1
// ),
// parts as (
// SELECT triggers.part
// SELECT extract.part
// FROM client as c
// JOIN documents as docs on docs.clientId = c.id
// JOIN documentCleans as clean on docs.id = clean.documentId
// JOIN documentTextTriggerExtractions as triggers
// on triggers.cleanId = clean.id
// WHERE date(triggers.createdAt) = date($2)
// JOIN documentTextExtractions as extract
// on extract.cleanId = clean.id
// WHERE date(extract.createdAt) = date($2)
// ),
// max_part as (
// SELECT COALESCE(max(part), 0) as max_part_num FROM parts
internal/database/repository/text_test.go
+53 -125
View File
@@ -16,17 +16,17 @@ import (
)
func TestTextExtraction(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, textup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer textup()
queries := cfg.GetDBQueries()
@@ -68,73 +68,31 @@ func TestTextExtraction(t *testing.T) {
require.NoError(t, err)
assert.False(t, isextract)
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanid,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
trigger, err := queries.GetDocumentTextTrigger(ctx, triggerId)
require.NoError(t, err)
assert.EqualExportedValues(t, &repository.GetDocumentTextTriggerRow{
ID: triggerId,
Cleanid: cleanid,
Version: 123,
Documentid: id,
}, trigger)
isextract, err = queries.IsDocumentTextExtracted(ctx, id)
require.NoError(t, err)
assert.False(t, isextract)
textractId := "hello"
err = queries.AddDocumentTextTriggerJobId(ctx, &repository.AddDocumentTextTriggerJobIdParams{
Textractjobid: &textractId,
ID: triggerId,
})
require.NoError(t, err)
trigger, err = queries.GetDocumentTextTrigger(ctx, triggerId)
require.NoError(t, err)
assert.EqualExportedValues(t, &repository.GetDocumentTextTriggerRow{
ID: triggerId,
Cleanid: cleanid,
Version: 123,
Textractjobid: &textractId,
Documentid: id,
}, trigger)
isextract, err = queries.IsDocumentTextExtracted(ctx, id)
require.NoError(t, err)
assert.False(t, isextract)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: bucket,
Key: key,
Hash: "example",
Part: 0,
Cleanid: cleanid,
Part: 0,
Bucket: bucket,
Key: key,
Hash: "example",
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
assert.NotNil(t, textId)
assert.NotEqual(t, uuid.UUID{}, textId)
isextract, err = queries.IsDocumentTextExtracted(ctx, id)
require.NoError(t, err)
assert.False(t, isextract)
isextract, err = queries.IsDocumentTextExtracted(ctx, id)
require.NoError(t, err)
assert.False(t, isextract)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Textid: textId,
Triggerid: triggerId,
Version: 543,
Textid: textId,
Version: 543,
})
require.NoError(t, err)
@@ -150,9 +108,6 @@ func TestTextExtraction(t *testing.T) {
assert.Equal(t, key, text.Key)
assert.Equal(t, "example", text.Hash)
assert.Equal(t, cleanid, text.Cleanid)
assert.Equal(t, triggerId, text.Triggerid)
assert.Equal(t, textractId, *text.Textractjobid)
assert.Equal(t, int64(123), text.Triggerversion)
assert.Equal(t, int64(543), text.Extractionversion)
assert.Equal(t, textId, text.ID)
@@ -171,17 +126,17 @@ func TestTextExtraction(t *testing.T) {
}
func TestTextTextractPart(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, textup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer textup()
queries := cfg.GetDBQueries()
@@ -232,9 +187,8 @@ func TestTextTextractPart(t *testing.T) {
Part: 0,
}, part)
_, err = queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
_, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Cleanid: cleanid,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
@@ -256,9 +210,8 @@ func TestTextTextractPart(t *testing.T) {
Part: 0,
}, part)
_, err = queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
_, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Cleanid: cleanid,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
@@ -280,9 +233,8 @@ func TestTextTextractPart(t *testing.T) {
Part: 0,
}, part)
_, err = queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
_, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Cleanid: cleanid,
Version: 123,
Part: 1,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
@@ -306,17 +258,17 @@ func TestTextTextractPart(t *testing.T) {
}
func TestTextOutPart(t *testing.T) {
t.Parallel()
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, textup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
net := test.DepNetwork.Get(t, ctx)
_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer textup()
queries := cfg.GetDBQueries()
@@ -354,9 +306,11 @@ func TestTextOutPart(t *testing.T) {
})
require.NoError(t, err)
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Cleanid: cleanid,
Version: 123,
Bucket: "hi",
Key: "hi",
Hash: "hi",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
@@ -364,6 +318,11 @@ func TestTextOutPart(t *testing.T) {
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Textid: textId,
Version: 123,
})
require.NoError(t, err)
part, err := queries.GetTextOutCurrentPart(ctx, &repository.GetTextOutCurrentPartParams{
Clientid: &clientId,
@@ -373,47 +332,17 @@ func TestTextOutPart(t *testing.T) {
},
})
require.NoError(t, err)
assert.Equal(t, &repository.GetTextOutCurrentPartRow{
Count: 0,
Part: 0,
}, part)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hi",
Hash: "hi",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Textid: textId,
Triggerid: triggerId,
Version: 123,
})
require.NoError(t, err)
part, err = queries.GetTextOutCurrentPart(ctx, &repository.GetTextOutCurrentPartParams{
Clientid: &clientId,
Querydate: pgtype.Timestamptz{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
assert.Equal(t, &repository.GetTextOutCurrentPartRow{
Count: 1,
Part: 0,
}, part)
textId, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hi",
Hash: "hi",
Part: 0,
Cleanid: cleanid,
Bucket: "hi",
Key: "hi",
Hash: "hi",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
@@ -421,9 +350,8 @@ func TestTextOutPart(t *testing.T) {
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Textid: textId,
Triggerid: triggerId,
Version: 123,
Textid: textId,
Version: 123,
})
require.NoError(t, err)
@@ -441,10 +369,11 @@ func TestTextOutPart(t *testing.T) {
}, part)
textId, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hi",
Hash: "hi",
Part: 0,
Cleanid: cleanid,
Bucket: "hi",
Key: "hi",
Hash: "hi",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
@@ -452,9 +381,8 @@ func TestTextOutPart(t *testing.T) {
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Textid: textId,
Triggerid: triggerId,
Version: 123,
Textid: textId,
Version: 123,
})
require.NoError(t, err)
@@ -472,10 +400,11 @@ func TestTextOutPart(t *testing.T) {
}, part)
textId, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hi",
Hash: "hi",
Part: 1,
Cleanid: cleanid,
Bucket: "hi",
Key: "hi",
Hash: "hi",
Part: 1,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
@@ -483,9 +412,8 @@ func TestTextOutPart(t *testing.T) {
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Textid: textId,
Triggerid: triggerId,
Version: 123,
Textid: textId,
Version: 123,
})
require.NoError(t, err)