Merged in feature/postprocessing (pull request #114)
Feature/postprocessing * tests * passtest * fixshorttests * mosttests * improvingbasedockerfile * testspeeds * testing * host * canparallel * clean * passfullsuite * singlepagemax * test * findfeatures * findstables * tbls * tablestoo * tablestoo * lateraltests * tableloc * cleanup * inlinetable * childids * cleanup * tests
This commit is contained in:
Michael McGuinness
@@ -14,12 +14,13 @@ import (
|
||||
|
||||
const addDocumentText = `-- name: AddDocumentText :one
|
||||
INSERT INTO documentTextExtractions
|
||||
(bucket, key, hash, createdAt, part)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
(cleanId, bucket, key, hash, createdAt, part)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
returning id
|
||||
`
|
||||
|
||||
type AddDocumentTextParams struct {
|
||||
Cleanid uuid.UUID `db:"cleanid"`
|
||||
Bucket string `db:"bucket"`
|
||||
Key string `db:"key"`
|
||||
Hash string `db:"hash"`
|
||||
@@ -30,11 +31,12 @@ type AddDocumentTextParams struct {
|
||||
// AddDocumentText
|
||||
//
|
||||
// INSERT INTO documentTextExtractions
|
||||
// (bucket, key, hash, createdAt, part)
|
||||
// VALUES ($1, $2, $3, $4, $5)
|
||||
// (cleanId, bucket, key, hash, createdAt, part)
|
||||
// VALUES ($1, $2, $3, $4, $5, $6)
|
||||
// returning id
|
||||
func (q *Queries) AddDocumentText(ctx context.Context, arg *AddDocumentTextParams) (uuid.UUID, error) {
|
||||
row := q.db.QueryRow(ctx, addDocumentText,
|
||||
arg.Cleanid,
|
||||
arg.Bucket,
|
||||
arg.Key,
|
||||
arg.Hash,
|
||||
@@ -48,76 +50,22 @@ func (q *Queries) AddDocumentText(ctx context.Context, arg *AddDocumentTextParam
|
||||
|
||||
const addDocumentTextEntry = `-- name: AddDocumentTextEntry :exec
|
||||
INSERT INTO documentTextExtractionEntries
|
||||
(textId, triggerId, version)
|
||||
VALUES ($1, $2, $3)
|
||||
(textId, version)
|
||||
VALUES ($1, $2)
|
||||
`
|
||||
|
||||
type AddDocumentTextEntryParams struct {
|
||||
Textid uuid.UUID `db:"textid"`
|
||||
Triggerid uuid.UUID `db:"triggerid"`
|
||||
Version int64 `db:"version"`
|
||||
Textid uuid.UUID `db:"textid"`
|
||||
Version int64 `db:"version"`
|
||||
}
|
||||
|
||||
// AddDocumentTextEntry
|
||||
//
|
||||
// INSERT INTO documentTextExtractionEntries
|
||||
// (textId, triggerId, version)
|
||||
// VALUES ($1, $2, $3)
|
||||
// (textId, version)
|
||||
// VALUES ($1, $2)
|
||||
func (q *Queries) AddDocumentTextEntry(ctx context.Context, arg *AddDocumentTextEntryParams) error {
|
||||
_, err := q.db.Exec(ctx, addDocumentTextEntry, arg.Textid, arg.Triggerid, arg.Version)
|
||||
return err
|
||||
}
|
||||
|
||||
const addDocumentTextTrigger = `-- name: AddDocumentTextTrigger :one
|
||||
INSERT INTO documentTextTriggerExtractions
|
||||
(cleanId, version, createdAt, part) VALUES
|
||||
($1, $2, $3, $4)
|
||||
returning id
|
||||
`
|
||||
|
||||
type AddDocumentTextTriggerParams struct {
|
||||
Cleanid uuid.UUID `db:"cleanid"`
|
||||
Version int64 `db:"version"`
|
||||
Createdat pgtype.Timestamp `db:"createdat"`
|
||||
Part uint16 `db:"part"`
|
||||
}
|
||||
|
||||
// AddDocumentTextTrigger
|
||||
//
|
||||
// INSERT INTO documentTextTriggerExtractions
|
||||
// (cleanId, version, createdAt, part) VALUES
|
||||
// ($1, $2, $3, $4)
|
||||
// returning id
|
||||
func (q *Queries) AddDocumentTextTrigger(ctx context.Context, arg *AddDocumentTextTriggerParams) (uuid.UUID, error) {
|
||||
row := q.db.QueryRow(ctx, addDocumentTextTrigger,
|
||||
arg.Cleanid,
|
||||
arg.Version,
|
||||
arg.Createdat,
|
||||
arg.Part,
|
||||
)
|
||||
var id uuid.UUID
|
||||
err := row.Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
|
||||
const addDocumentTextTriggerJobId = `-- name: AddDocumentTextTriggerJobId :exec
|
||||
UPDATE documentTextTriggerExtractions
|
||||
SET textractJobId = $1
|
||||
WHERE id = $2
|
||||
`
|
||||
|
||||
type AddDocumentTextTriggerJobIdParams struct {
|
||||
Textractjobid *string `db:"textractjobid"`
|
||||
ID uuid.UUID `db:"id"`
|
||||
}
|
||||
|
||||
// AddDocumentTextTriggerJobId
|
||||
//
|
||||
// UPDATE documentTextTriggerExtractions
|
||||
// SET textractJobId = $1
|
||||
// WHERE id = $2
|
||||
func (q *Queries) AddDocumentTextTriggerJobId(ctx context.Context, arg *AddDocumentTextTriggerJobIdParams) error {
|
||||
_, err := q.db.Exec(ctx, addDocumentTextTriggerJobId, arg.Textractjobid, arg.ID)
|
||||
_, err := q.db.Exec(ctx, addDocumentTextEntry, arg.Textid, arg.Version)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -140,51 +88,15 @@ func (q *Queries) GetDocumentTextExtractionByHash(ctx context.Context, arg *GetD
|
||||
return id, err
|
||||
}
|
||||
|
||||
const getDocumentTextTrigger = `-- name: GetDocumentTextTrigger :one
|
||||
SELECT dt.id, dt.cleanId, dt.version, dt.textractJobId, dc.documentId
|
||||
from documentTextTriggerExtractions as dt
|
||||
join documentCleans as dc on dt.cleanId = dc.id
|
||||
where dt.id = $1
|
||||
`
|
||||
|
||||
type GetDocumentTextTriggerRow struct {
|
||||
ID uuid.UUID `db:"id"`
|
||||
Cleanid uuid.UUID `db:"cleanid"`
|
||||
Version int64 `db:"version"`
|
||||
Textractjobid *string `db:"textractjobid"`
|
||||
Documentid uuid.UUID `db:"documentid"`
|
||||
}
|
||||
|
||||
// GetDocumentTextTrigger
|
||||
//
|
||||
// SELECT dt.id, dt.cleanId, dt.version, dt.textractJobId, dc.documentId
|
||||
// from documentTextTriggerExtractions as dt
|
||||
// join documentCleans as dc on dt.cleanId = dc.id
|
||||
// where dt.id = $1
|
||||
func (q *Queries) GetDocumentTextTrigger(ctx context.Context, triggerid uuid.UUID) (*GetDocumentTextTriggerRow, error) {
|
||||
row := q.db.QueryRow(ctx, getDocumentTextTrigger, triggerid)
|
||||
var i GetDocumentTextTriggerRow
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.Cleanid,
|
||||
&i.Version,
|
||||
&i.Textractjobid,
|
||||
&i.Documentid,
|
||||
)
|
||||
return &i, err
|
||||
}
|
||||
|
||||
const getTextEntryByDocId = `-- name: GetTextEntryByDocId :one
|
||||
SELECT id, documentId, bucket, key, hash, cleanId,
|
||||
triggerId, textractJobId, triggerVersion, extractionVersion
|
||||
SELECT id, documentId, bucket, key, hash, cleanId, extractionVersion
|
||||
FROM currentTextEntries
|
||||
WHERE documentId = $1
|
||||
`
|
||||
|
||||
// GetTextEntryByDocId
|
||||
//
|
||||
// SELECT id, documentId, bucket, key, hash, cleanId,
|
||||
// triggerId, textractJobId, triggerVersion, extractionVersion
|
||||
// SELECT id, documentId, bucket, key, hash, cleanId, extractionVersion
|
||||
// FROM currentTextEntries
|
||||
// WHERE documentId = $1
|
||||
func (q *Queries) GetTextEntryByDocId(ctx context.Context, documentid uuid.UUID) (*Currenttextentry, error) {
|
||||
@@ -197,9 +109,6 @@ func (q *Queries) GetTextEntryByDocId(ctx context.Context, documentid uuid.UUID)
|
||||
&i.Key,
|
||||
&i.Hash,
|
||||
&i.Cleanid,
|
||||
&i.Triggerid,
|
||||
&i.Textractjobid,
|
||||
&i.Triggerversion,
|
||||
&i.Extractionversion,
|
||||
)
|
||||
return &i, err
|
||||
@@ -210,17 +119,12 @@ WITH client as (
|
||||
select id from clients where id = $1
|
||||
),
|
||||
parts as (
|
||||
SELECT extractions.part
|
||||
SELECT extract.part
|
||||
FROM client as c
|
||||
JOIN documents as docs on docs.clientId = c.id
|
||||
JOIN documentCleans as clean on docs.id = clean.documentId
|
||||
JOIN documentTextTriggerExtractions as triggers
|
||||
on triggers.cleanId = clean.id
|
||||
JOIN documentTextExtractionEntries as textEntries
|
||||
on textEntries.triggerId = triggers.id
|
||||
JOIN documentTextExtractions as extractions
|
||||
on extractions.id = textEntries.textId
|
||||
WHERE date(extractions.createdAt) = date($2)
|
||||
JOIN documentTextExtractions extract ON extract.cleanId = clean.id
|
||||
WHERE date(extract.createdAt) = date($2)
|
||||
),
|
||||
max_part as (
|
||||
SELECT COALESCE(max(part), 0) as max_part_num FROM parts
|
||||
@@ -249,17 +153,12 @@ type GetTextOutCurrentPartRow struct {
|
||||
// select id from clients where id = $1
|
||||
// ),
|
||||
// parts as (
|
||||
// SELECT extractions.part
|
||||
// SELECT extract.part
|
||||
// FROM client as c
|
||||
// JOIN documents as docs on docs.clientId = c.id
|
||||
// JOIN documentCleans as clean on docs.id = clean.documentId
|
||||
// JOIN documentTextTriggerExtractions as triggers
|
||||
// on triggers.cleanId = clean.id
|
||||
// JOIN documentTextExtractionEntries as textEntries
|
||||
// on textEntries.triggerId = triggers.id
|
||||
// JOIN documentTextExtractions as extractions
|
||||
// on extractions.id = textEntries.textId
|
||||
// WHERE date(extractions.createdAt) = date($2)
|
||||
// JOIN documentTextExtractions extract ON extract.cleanId = clean.id
|
||||
// WHERE date(extract.createdAt) = date($2)
|
||||
// ),
|
||||
// max_part as (
|
||||
// SELECT COALESCE(max(part), 0) as max_part_num FROM parts
|
||||
@@ -282,13 +181,13 @@ WITH client as (
|
||||
select id from clients where id = $1
|
||||
),
|
||||
parts as (
|
||||
SELECT triggers.part
|
||||
SELECT extract.part
|
||||
FROM client as c
|
||||
JOIN documents as docs on docs.clientId = c.id
|
||||
JOIN documentCleans as clean on docs.id = clean.documentId
|
||||
JOIN documentTextTriggerExtractions as triggers
|
||||
on triggers.cleanId = clean.id
|
||||
WHERE date(triggers.createdAt) = date($2)
|
||||
JOIN documentTextExtractions as extract
|
||||
on extract.cleanId = clean.id
|
||||
WHERE date(extract.createdAt) = date($2)
|
||||
),
|
||||
max_part as (
|
||||
SELECT COALESCE(max(part), 0) as max_part_num FROM parts
|
||||
@@ -317,13 +216,13 @@ type GetTextractOutputCurrentPartRow struct {
|
||||
// select id from clients where id = $1
|
||||
// ),
|
||||
// parts as (
|
||||
// SELECT triggers.part
|
||||
// SELECT extract.part
|
||||
// FROM client as c
|
||||
// JOIN documents as docs on docs.clientId = c.id
|
||||
// JOIN documentCleans as clean on docs.id = clean.documentId
|
||||
// JOIN documentTextTriggerExtractions as triggers
|
||||
// on triggers.cleanId = clean.id
|
||||
// WHERE date(triggers.createdAt) = date($2)
|
||||
// JOIN documentTextExtractions as extract
|
||||
// on extract.cleanId = clean.id
|
||||
// WHERE date(extract.createdAt) = date($2)
|
||||
// ),
|
||||
// max_part as (
|
||||
// SELECT COALESCE(max(part), 0) as max_part_num FROM parts
|
||||
|
||||
Reference in New Issue
Block a user