jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/part (pull request #113)

Browse Source 
Size Import Parts

* parts

* think

* workingpart

* textout
This commit is contained in:
Michael McGuinness
2025-04-08 12:44:09 +00:00
parent 60812aba91
commit edc50c7510
21 changed files with 873 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/migrations/00000000000001_extensions.up.sql
+5 -2
View File
@@ -27,8 +27,11 @@ CREATE FUNCTION isInVersion(
RETURNS boolean as $$
BEGIN
RETURN _version is null OR (
_version >= _addedVersion AND
_version >= _addedVersion AND
(_removedVersion is null OR _version < _removedVersion)
);
END;
$$ LANGUAGE plpgsql;
$$ LANGUAGE plpgsql;
CREATE DOMAIN unsignedsmallint AS SMALLINT
CHECK (VALUE >= 0);
internal/database/migrations/00000000000005_documents.up.sql
+5 -1
View File
@@ -54,6 +54,8 @@ CREATE TABLE documentTextTriggerExtractions (
id uuid primary key DEFAULT uuid_generate_v7(),
cleanId uuid not null,
version bigint not null,
createdAt timestamp not null,
part unsignedsmallint not null,
textractJobId text,
foreign key (cleanId) references documentCleans(id)
);
@@ -62,7 +64,9 @@ CREATE TABLE documentTextExtractions (
id uuid primary key DEFAULT uuid_generate_v7(),
bucket text not null,
key text not null,
hash text not null
hash text not null,
createdAt timestamp not null,
part unsignedsmallint not null
);
CREATE TABLE documentTextExtractionEntries (
internal/database/queries/text.sql
+64 -4
View File
@@ -7,10 +7,65 @@ SELECT EXISTS(
SELECT id FROM currentTextEntries WHERE cleanId = @cleanEntryId and hash = @hash;
-- name: AddDocumentTextTrigger :one
INSERT INTO documentTextTriggerExtractions (cleanId, version) VALUES ($1, $2) returning id;
INSERT INTO documentTextTriggerExtractions
(cleanId, version, createdAt, part) VALUES
($1, $2, $3, $4)
returning id;
-- name: GetTextractOutputCurrentPart :one
WITH client as (
select id from clients where id = @clientId
),
parts as (
SELECT triggers.part
FROM client as c
JOIN documents as docs on docs.clientId = c.id
JOIN documentCleans as clean on docs.id = clean.documentId
JOIN documentTextTriggerExtractions as triggers
on triggers.cleanId = clean.id
WHERE date(triggers.createdAt) = date(@queryDate)
),
max_part as (
SELECT COALESCE(max(part), 0) as max_part_num FROM parts
)
SELECT
COALESCE(p.part, 0)::unsignedsmallint as part,
COUNT(p.part) as count
FROM parts p
RIGHT JOIN max_part mp ON p.part = mp.max_part_num
GROUP BY p.part;
-- name: GetTextOutCurrentPart :one
WITH client as (
select id from clients where id = @clientId
),
parts as (
SELECT extractions.part
FROM client as c
JOIN documents as docs on docs.clientId = c.id
JOIN documentCleans as clean on docs.id = clean.documentId
JOIN documentTextTriggerExtractions as triggers
on triggers.cleanId = clean.id
JOIN documentTextExtractionEntries as textEntries
on textEntries.triggerId = triggers.id
JOIN documentTextExtractions as extractions
on extractions.id = textEntries.textId
WHERE date(extractions.createdAt) = date(@queryDate)
),
max_part as (
SELECT COALESCE(max(part), 0) as max_part_num FROM parts
)
SELECT
COALESCE(p.part, 0)::unsignedsmallint as part,
COUNT(p.part) as count
FROM parts p
RIGHT JOIN max_part mp ON p.part = mp.max_part_num
GROUP BY p.part;
-- name: AddDocumentTextTriggerJobId :exec
UPDATE documentTextTriggerExtractions SET textractJobId = @textractJobId WHERE id = @id;
UPDATE documentTextTriggerExtractions
SET textractJobId = @textractJobId
WHERE id = @id;
-- name: GetDocumentTextTrigger :one
SELECT dt.id, dt.cleanId, dt.version, dt.textractJobId, dc.documentId
@@ -19,10 +74,15 @@ SELECT dt.id, dt.cleanId, dt.version, dt.textractJobId, dc.documentId
where dt.id = @triggerId;
-- name: AddDocumentText :one
INSERT INTO documentTextExtractions (bucket, key, hash) VALUES ($1, $2, $3) returning id;
INSERT INTO documentTextExtractions
(bucket, key, hash, createdAt, part)
VALUES ($1, $2, $3, $4, $5)
returning id;
-- name: AddDocumentTextEntry :exec
INSERT INTO documentTextExtractionEntries (textId, triggerId, version) VALUES ($1, $2, $3);
INSERT INTO documentTextExtractionEntries
(textId, triggerId, version)
VALUES ($1, $2, $3);
-- name: GetTextEntryByDocId :one
SELECT id, documentId, bucket, key, hash, cleanId,
internal/database/repository/models.go
+12 -8
View File
@@ -326,10 +326,12 @@ type Documententry struct {
}
type Documenttextextraction struct {
ID uuid.UUID `db:"id"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Hash string `db:"hash"`
ID uuid.UUID `db:"id"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Hash string `db:"hash"`
Createdat pgtype.Timestamp `db:"createdat"`
Part uint16 `db:"part"`
}
type Documenttextextractionentry struct {
@@ -340,10 +342,12 @@ type Documenttextextractionentry struct {
}
type Documenttexttriggerextraction struct {
ID uuid.UUID `db:"id"`
Cleanid uuid.UUID `db:"cleanid"`
Version int64 `db:"version"`
Textractjobid *string `db:"textractjobid"`
ID uuid.UUID `db:"id"`
Cleanid uuid.UUID `db:"cleanid"`
Version int64 `db:"version"`
Createdat pgtype.Timestamp `db:"createdat"`
Part uint16 `db:"part"`
Textractjobid *string `db:"textractjobid"`
}
type Fullactivecollector struct {
internal/database/repository/result_test.go
+42
View File
@@ -3,12 +3,14 @@ package repository_test
import (
"context"
"testing"
"time"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/test"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -98,12 +100,22 @@ func TestResults(t *testing.T) {
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanid,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
Hash: "example",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
@@ -265,12 +277,22 @@ func TestResultValues(t *testing.T) {
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanid,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
Hash: "example",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
@@ -479,11 +501,21 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanid,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
@@ -559,11 +591,21 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
triggerTwoId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleantwoid,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
textTwoId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
internal/database/repository/sync_test.go
+32
View File
@@ -3,11 +3,13 @@ package repository_test
import (
"context"
"testing"
"time"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/test"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -286,12 +288,22 @@ func TestClientSync(t *testing.T) {
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanId,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
Hash: "example",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
@@ -458,11 +470,21 @@ func TestClientSync(t *testing.T) {
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanId,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
@@ -575,11 +597,21 @@ func TestClientSync(t *testing.T) {
triggerThreeId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanthreeid,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
textThreeId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hello",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
internal/database/repository/text.sql.go
+187 -15
View File
@@ -9,30 +9,47 @@ import (
"context"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
)
const addDocumentText = `-- name: AddDocumentText :one
INSERT INTO documentTextExtractions (bucket, key, hash) VALUES ($1, $2, $3) returning id
INSERT INTO documentTextExtractions
(bucket, key, hash, createdAt, part)
VALUES ($1, $2, $3, $4, $5)
returning id
`
type AddDocumentTextParams struct {
Bucket string `db:"bucket"`
Key string `db:"key"`
Hash string `db:"hash"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Hash string `db:"hash"`
Createdat pgtype.Timestamp `db:"createdat"`
Part uint16 `db:"part"`
}
// AddDocumentText
//
// INSERT INTO documentTextExtractions (bucket, key, hash) VALUES ($1, $2, $3) returning id
// INSERT INTO documentTextExtractions
// (bucket, key, hash, createdAt, part)
// VALUES ($1, $2, $3, $4, $5)
// returning id
func (q *Queries) AddDocumentText(ctx context.Context, arg *AddDocumentTextParams) (uuid.UUID, error) {
row := q.db.QueryRow(ctx, addDocumentText, arg.Bucket, arg.Key, arg.Hash)
row := q.db.QueryRow(ctx, addDocumentText,
arg.Bucket,
arg.Key,
arg.Hash,
arg.Createdat,
arg.Part,
)
var id uuid.UUID
err := row.Scan(&id)
return id, err
}
const addDocumentTextEntry = `-- name: AddDocumentTextEntry :exec
INSERT INTO documentTextExtractionEntries (textId, triggerId, version) VALUES ($1, $2, $3)
INSERT INTO documentTextExtractionEntries
(textId, triggerId, version)
VALUES ($1, $2, $3)
`
type AddDocumentTextEntryParams struct {
@@ -43,33 +60,50 @@ type AddDocumentTextEntryParams struct {
// AddDocumentTextEntry
//
// INSERT INTO documentTextExtractionEntries (textId, triggerId, version) VALUES ($1, $2, $3)
// INSERT INTO documentTextExtractionEntries
// (textId, triggerId, version)
// VALUES ($1, $2, $3)
func (q *Queries) AddDocumentTextEntry(ctx context.Context, arg *AddDocumentTextEntryParams) error {
_, err := q.db.Exec(ctx, addDocumentTextEntry, arg.Textid, arg.Triggerid, arg.Version)
return err
}
const addDocumentTextTrigger = `-- name: AddDocumentTextTrigger :one
INSERT INTO documentTextTriggerExtractions (cleanId, version) VALUES ($1, $2) returning id
INSERT INTO documentTextTriggerExtractions
(cleanId, version, createdAt, part) VALUES
($1, $2, $3, $4)
returning id
`
type AddDocumentTextTriggerParams struct {
Cleanid uuid.UUID `db:"cleanid"`
Version int64 `db:"version"`
Cleanid uuid.UUID `db:"cleanid"`
Version int64 `db:"version"`
Createdat pgtype.Timestamp `db:"createdat"`
Part uint16 `db:"part"`
}
// AddDocumentTextTrigger
//
// INSERT INTO documentTextTriggerExtractions (cleanId, version) VALUES ($1, $2) returning id
// INSERT INTO documentTextTriggerExtractions
// (cleanId, version, createdAt, part) VALUES
// ($1, $2, $3, $4)
// returning id
func (q *Queries) AddDocumentTextTrigger(ctx context.Context, arg *AddDocumentTextTriggerParams) (uuid.UUID, error) {
row := q.db.QueryRow(ctx, addDocumentTextTrigger, arg.Cleanid, arg.Version)
row := q.db.QueryRow(ctx, addDocumentTextTrigger,
arg.Cleanid,
arg.Version,
arg.Createdat,
arg.Part,
)
var id uuid.UUID
err := row.Scan(&id)
return id, err
}
const addDocumentTextTriggerJobId = `-- name: AddDocumentTextTriggerJobId :exec
UPDATE documentTextTriggerExtractions SET textractJobId = $1 WHERE id = $2
UPDATE documentTextTriggerExtractions
SET textractJobId = $1
WHERE id = $2
`
type AddDocumentTextTriggerJobIdParams struct {
@@ -79,7 +113,9 @@ type AddDocumentTextTriggerJobIdParams struct {
// AddDocumentTextTriggerJobId
//
// UPDATE documentTextTriggerExtractions SET textractJobId = $1 WHERE id = $2
// UPDATE documentTextTriggerExtractions
// SET textractJobId = $1
// WHERE id = $2
func (q *Queries) AddDocumentTextTriggerJobId(ctx context.Context, arg *AddDocumentTextTriggerJobIdParams) error {
_, err := q.db.Exec(ctx, addDocumentTextTriggerJobId, arg.Textractjobid, arg.ID)
return err
@@ -169,6 +205,142 @@ func (q *Queries) GetTextEntryByDocId(ctx context.Context, documentid uuid.UUID)
return &i, err
}
const getTextOutCurrentPart = `-- name: GetTextOutCurrentPart :one
WITH client as (
select id from clients where id = $1
),
parts as (
SELECT extractions.part
FROM client as c
JOIN documents as docs on docs.clientId = c.id
JOIN documentCleans as clean on docs.id = clean.documentId
JOIN documentTextTriggerExtractions as triggers
on triggers.cleanId = clean.id
JOIN documentTextExtractionEntries as textEntries
on textEntries.triggerId = triggers.id
JOIN documentTextExtractions as extractions
on extractions.id = textEntries.textId
WHERE date(extractions.createdAt) = date($2)
),
max_part as (
SELECT COALESCE(max(part), 0) as max_part_num FROM parts
)
SELECT
COALESCE(p.part, 0)::unsignedsmallint as part,
COUNT(p.part) as count
FROM parts p
RIGHT JOIN max_part mp ON p.part = mp.max_part_num
GROUP BY p.part
`
type GetTextOutCurrentPartParams struct {
Clientid *string `db:"clientid"`
Querydate pgtype.Timestamptz `db:"querydate"`
}
type GetTextOutCurrentPartRow struct {
Part uint16 `db:"part"`
Count int64 `db:"count"`
}
// GetTextOutCurrentPart
//
// WITH client as (
// select id from clients where id = $1
// ),
// parts as (
// SELECT extractions.part
// FROM client as c
// JOIN documents as docs on docs.clientId = c.id
// JOIN documentCleans as clean on docs.id = clean.documentId
// JOIN documentTextTriggerExtractions as triggers
// on triggers.cleanId = clean.id
// JOIN documentTextExtractionEntries as textEntries
// on textEntries.triggerId = triggers.id
// JOIN documentTextExtractions as extractions
// on extractions.id = textEntries.textId
// WHERE date(extractions.createdAt) = date($2)
// ),
// max_part as (
// SELECT COALESCE(max(part), 0) as max_part_num FROM parts
// )
// SELECT
// COALESCE(p.part, 0)::unsignedsmallint as part,
// COUNT(p.part) as count
// FROM parts p
// RIGHT JOIN max_part mp ON p.part = mp.max_part_num
// GROUP BY p.part
func (q *Queries) GetTextOutCurrentPart(ctx context.Context, arg *GetTextOutCurrentPartParams) (*GetTextOutCurrentPartRow, error) {
row := q.db.QueryRow(ctx, getTextOutCurrentPart, arg.Clientid, arg.Querydate)
var i GetTextOutCurrentPartRow
err := row.Scan(&i.Part, &i.Count)
return &i, err
}
const getTextractOutputCurrentPart = `-- name: GetTextractOutputCurrentPart :one
WITH client as (
select id from clients where id = $1
),
parts as (
SELECT triggers.part
FROM client as c
JOIN documents as docs on docs.clientId = c.id
JOIN documentCleans as clean on docs.id = clean.documentId
JOIN documentTextTriggerExtractions as triggers
on triggers.cleanId = clean.id
WHERE date(triggers.createdAt) = date($2)
),
max_part as (
SELECT COALESCE(max(part), 0) as max_part_num FROM parts
)
SELECT
COALESCE(p.part, 0)::unsignedsmallint as part,
COUNT(p.part) as count
FROM parts p
RIGHT JOIN max_part mp ON p.part = mp.max_part_num
GROUP BY p.part
`
type GetTextractOutputCurrentPartParams struct {
Clientid *string `db:"clientid"`
Querydate pgtype.Timestamptz `db:"querydate"`
}
type GetTextractOutputCurrentPartRow struct {
Part uint16 `db:"part"`
Count int64 `db:"count"`
}
// GetTextractOutputCurrentPart
//
// WITH client as (
// select id from clients where id = $1
// ),
// parts as (
// SELECT triggers.part
// FROM client as c
// JOIN documents as docs on docs.clientId = c.id
// JOIN documentCleans as clean on docs.id = clean.documentId
// JOIN documentTextTriggerExtractions as triggers
// on triggers.cleanId = clean.id
// WHERE date(triggers.createdAt) = date($2)
// ),
// max_part as (
// SELECT COALESCE(max(part), 0) as max_part_num FROM parts
// )
// SELECT
// COALESCE(p.part, 0)::unsignedsmallint as part,
// COUNT(p.part) as count
// FROM parts p
// RIGHT JOIN max_part mp ON p.part = mp.max_part_num
// GROUP BY p.part
func (q *Queries) GetTextractOutputCurrentPart(ctx context.Context, arg *GetTextractOutputCurrentPartParams) (*GetTextractOutputCurrentPartRow, error) {
row := q.db.QueryRow(ctx, getTextractOutputCurrentPart, arg.Clientid, arg.Querydate)
var i GetTextractOutputCurrentPartRow
err := row.Scan(&i.Part, &i.Count)
return &i, err
}
const isDocumentTextExtracted = `-- name: IsDocumentTextExtracted :one
SELECT EXISTS(
SELECT 1 FROM currentTextEntries WHERE documentId = $1
internal/database/repository/text_test.go
+345
View File
@@ -3,12 +3,14 @@ package repository_test
import (
"context"
"testing"
"time"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/test"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -69,6 +71,11 @@ func TestTextExtraction(t *testing.T) {
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanid,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
@@ -110,6 +117,11 @@ func TestTextExtraction(t *testing.T) {
Bucket: bucket,
Key: key,
Hash: "example",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
assert.NotNil(t, textId)
@@ -157,3 +169,336 @@ func TestTextExtraction(t *testing.T) {
})
require.EqualError(t, err, "no rows in result set")
}
func TestTextTextractPart(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, textup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer textup()
queries := cfg.GetDBQueries()
clientId := "EXAMPLE"
err := queries.CreateClient(ctx, &repository.CreateClientParams{
Name: "example_client",
ID: clientId,
})
require.NoError(t, err)
hash := "example_hash"
id, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientId,
Hash: hash,
})
require.NoError(t, err)
assert.NotEmpty(t, id)
bucket := "example_bucket"
key := "example_key"
cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
Documentid: id,
Bucket: &bucket,
Key: &key,
Hash: &hash,
Mimetype: repository.NullCleanmimetype{
Valid: true,
Cleanmimetype: repository.CleanmimetypeApplicationPdf,
},
})
require.NoError(t, err)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Cleanid: cleanid,
Version: 1,
})
require.NoError(t, err)
part, err := queries.GetTextractOutputCurrentPart(ctx, &repository.GetTextractOutputCurrentPartParams{
Clientid: &clientId,
Querydate: pgtype.Timestamptz{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
assert.Equal(t, &repository.GetTextractOutputCurrentPartRow{
Count: 0,
Part: 0,
}, part)
_, err = queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanid,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
part, err = queries.GetTextractOutputCurrentPart(ctx, &repository.GetTextractOutputCurrentPartParams{
Clientid: &clientId,
Querydate: pgtype.Timestamptz{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
assert.Equal(t, &repository.GetTextractOutputCurrentPartRow{
Count: 1,
Part: 0,
}, part)
_, err = queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanid,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
part, err = queries.GetTextractOutputCurrentPart(ctx, &repository.GetTextractOutputCurrentPartParams{
Clientid: &clientId,
Querydate: pgtype.Timestamptz{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
assert.Equal(t, &repository.GetTextractOutputCurrentPartRow{
Count: 2,
Part: 0,
}, part)
_, err = queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanid,
Version: 123,
Part: 1,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
part, err = queries.GetTextractOutputCurrentPart(ctx, &repository.GetTextractOutputCurrentPartParams{
Clientid: &clientId,
Querydate: pgtype.Timestamptz{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
assert.Equal(t, &repository.GetTextractOutputCurrentPartRow{
Count: 1,
Part: 1,
}, part)
}
func TestTextOutPart(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
_, textup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
RunMigrations: true,
})
defer textup()
queries := cfg.GetDBQueries()
clientId := "EXAMPLE"
err := queries.CreateClient(ctx, &repository.CreateClientParams{
Name: "example_client",
ID: clientId,
})
require.NoError(t, err)
hash := "example_hash"
id, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientId,
Hash: hash,
})
require.NoError(t, err)
assert.NotEmpty(t, id)
bucket := "example_bucket"
key := "example_key"
cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
Documentid: id,
Bucket: &bucket,
Key: &key,
Hash: &hash,
Mimetype: repository.NullCleanmimetype{
Valid: true,
Cleanmimetype: repository.CleanmimetypeApplicationPdf,
},
})
require.NoError(t, err)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Cleanid: cleanid,
Version: 1,
})
require.NoError(t, err)
triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
Cleanid: cleanid,
Version: 123,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
part, err := queries.GetTextOutCurrentPart(ctx, &repository.GetTextOutCurrentPartParams{
Clientid: &clientId,
Querydate: pgtype.Timestamptz{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
assert.Equal(t, &repository.GetTextOutCurrentPartRow{
Count: 0,
Part: 0,
}, part)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hi",
Hash: "hi",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Textid: textId,
Triggerid: triggerId,
Version: 123,
})
require.NoError(t, err)
part, err = queries.GetTextOutCurrentPart(ctx, &repository.GetTextOutCurrentPartParams{
Clientid: &clientId,
Querydate: pgtype.Timestamptz{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
assert.Equal(t, &repository.GetTextOutCurrentPartRow{
Count: 1,
Part: 0,
}, part)
textId, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hi",
Hash: "hi",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Textid: textId,
Triggerid: triggerId,
Version: 123,
})
require.NoError(t, err)
part, err = queries.GetTextOutCurrentPart(ctx, &repository.GetTextOutCurrentPartParams{
Clientid: &clientId,
Querydate: pgtype.Timestamptz{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
assert.Equal(t, &repository.GetTextOutCurrentPartRow{
Count: 2,
Part: 0,
}, part)
textId, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hi",
Hash: "hi",
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Textid: textId,
Triggerid: triggerId,
Version: 123,
})
require.NoError(t, err)
part, err = queries.GetTextOutCurrentPart(ctx, &repository.GetTextOutCurrentPartParams{
Clientid: &clientId,
Querydate: pgtype.Timestamptz{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
assert.Equal(t, &repository.GetTextOutCurrentPartRow{
Count: 3,
Part: 0,
}, part)
textId, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
Key: "hi",
Hash: "hi",
Part: 1,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Textid: textId,
Triggerid: triggerId,
Version: 123,
})
require.NoError(t, err)
part, err = queries.GetTextOutCurrentPart(ctx, &repository.GetTextOutCurrentPartParams{
Clientid: &clientId,
Querydate: pgtype.Timestamptz{
Time: time.Now().UTC(),
Valid: true,
},
})
require.NoError(t, err)
assert.Equal(t, &repository.GetTextOutCurrentPartRow{
Count: 1,
Part: 1,
}, part)
}