jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/clean (pull request #76)

Browse Source 
Clean Set Up

* fail

* starteddb

* constraint

* cleantest

* fixqueries

* fixtests

* storemimetype

* buffer

* tests

* setup

* passingtests

* pdfHElloWorld

* rm

* test

* notodos

* tests

* clean

* testpdf
This commit is contained in:
Michael McGuinness
2025-02-28 13:11:53 +00:00
parent a49084ebce
commit f11f4def43
198 changed files with 71722 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/repository/clean.sql.go
+18 -12
View File
@@ -12,38 +12,42 @@ import (
)
const addDocumentCleanEntry = `-- name: AddDocumentCleanEntry :exec
INSERT INTO documentCleans (documentId, version, bucket, key) VALUES ($1, $2, $3, $4)
INSERT INTO documentCleans (documentId, version, bucket, key, mimetype, fail) VALUES ($1, $2, $3, $4, $5, $6)
`
type AddDocumentCleanEntryParams struct {
Documentid pgtype.UUID `db:"documentid"`
Version int32 `db:"version"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Documentid pgtype.UUID `db:"documentid"`
Version int32 `db:"version"`
Bucket *string `db:"bucket"`
Key *string `db:"key"`
Mimetype NullCleanmimetypes `db:"mimetype"`
Fail NullCleanfailtype `db:"fail"`
}
// AddDocumentCleanEntry
//
// INSERT INTO documentCleans (documentId, version, bucket, key) VALUES ($1, $2, $3, $4)
// INSERT INTO documentCleans (documentId, version, bucket, key, mimetype, fail) VALUES ($1, $2, $3, $4, $5, $6)
func (q *Queries) AddDocumentCleanEntry(ctx context.Context, arg *AddDocumentCleanEntryParams) error {
_, err := q.db.Exec(ctx, addDocumentCleanEntry,
arg.Documentid,
arg.Version,
arg.Bucket,
arg.Key,
arg.Mimetype,
arg.Fail,
)
return err
}
const getDocumentCleanEntry = `-- name: GetDocumentCleanEntry :one
SELECT id, documentId, bucket, key, version
SELECT id, documentId, bucket, key, version, mimetype, fail
FROM currentCleanEntries
WHERE documentId = $1
`
// GetDocumentCleanEntry
//
// SELECT id, documentId, bucket, key, version
// SELECT id, documentId, bucket, key, version, mimetype, fail
// FROM currentCleanEntries
// WHERE documentId = $1
func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.UUID) (*Currentcleanentry, error) {
@@ -55,23 +59,25 @@ func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.U
&i.Bucket,
&i.Key,
&i.Version,
&i.Mimetype,
&i.Fail,
)
return &i, err
}
const isDocumentClean = `-- name: IsDocumentClean :one
const hasDocumentCleanEntry = `-- name: HasDocumentCleanEntry :one
SELECT EXISTS(
SELECT 1 FROM currentCleanEntries WHERE documentId = $1
)
`
// IsDocumentClean
// HasDocumentCleanEntry
//
// SELECT EXISTS(
// SELECT 1 FROM currentCleanEntries WHERE documentId = $1
// )
func (q *Queries) IsDocumentClean(ctx context.Context, documentid pgtype.UUID) (bool, error) {
row := q.db.QueryRow(ctx, isDocumentClean, documentid)
func (q *Queries) HasDocumentCleanEntry(ctx context.Context, documentid pgtype.UUID) (bool, error) {
row := q.db.QueryRow(ctx, hasDocumentCleanEntry, documentid)
var exists bool
err := row.Scan(&exists)
return exists, err
internal/database/repository/clean_test.go
+48 -6
View File
@@ -43,7 +43,7 @@ func TestClean(t *testing.T) {
assert.NoError(t, err)
assert.NotEmpty(t, id)
isclean, err := queries.IsDocumentClean(ctx, id)
isclean, err := queries.HasDocumentCleanEntry(ctx, id)
assert.NoError(t, err)
assert.False(t, isclean)
@@ -52,24 +52,66 @@ func TestClean(t *testing.T) {
bucket := "example_bucket"
key := "example_key"
mimetype := repository.NullCleanmimetypes{
Valid: true,
Cleanmimetypes: repository.CleanmimetypesApplicationPdf,
}
fail := repository.NullCleanfailtype{
Valid: true,
Cleanfailtype: repository.CleanfailtypeInvalidMimetype,
}
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: id,
Version: 1,
Bucket: bucket,
Key: key,
Bucket: &bucket,
Key: &key,
Mimetype: mimetype,
Fail: fail,
})
assert.Error(t, err)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: id,
Version: 1,
Fail: fail,
})
assert.NoError(t, err)
isclean, err = queries.IsDocumentClean(ctx, id)
isclean, err = queries.HasDocumentCleanEntry(ctx, id)
assert.NoError(t, err)
assert.True(t, isclean)
clean, err := queries.GetDocumentCleanEntry(ctx, id)
assert.NoError(t, err)
assert.Equal(t, id, clean.Documentid)
assert.Equal(t, bucket, clean.Bucket)
assert.Equal(t, key, clean.Key)
assert.Nil(t, clean.Bucket)
assert.Nil(t, clean.Key)
assert.Equal(t, fail, clean.Fail)
assert.Equal(t, int32(1), clean.Version)
assert.NotEqual(t, pgtype.UUID{}, clean.ID)
assert.True(t, clean.ID.Valid)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: id,
Version: 2,
Bucket: &bucket,
Key: &key,
Mimetype: mimetype,
})
assert.NoError(t, err)
isclean, err = queries.HasDocumentCleanEntry(ctx, id)
assert.NoError(t, err)
assert.True(t, isclean)
clean, err = queries.GetDocumentCleanEntry(ctx, id)
assert.NoError(t, err)
assert.Equal(t, id, clean.Documentid)
assert.Equal(t, bucket, *clean.Bucket)
assert.Equal(t, key, *clean.Key)
assert.Equal(t, mimetype, clean.Mimetype)
assert.False(t, clean.Fail.Valid)
assert.Equal(t, int32(2), clean.Version)
assert.NotEqual(t, pgtype.UUID{}, clean.ID)
assert.True(t, clean.ID.Valid)
}
internal/database/repository/job.sql.go
+36 -14
View File
@@ -74,26 +74,37 @@ docs AS (
-- Get all documents for this job
SELECT id, jobId FROM documents WHERE jobId = $1
),
doc_clean_entries as (
-- Documents with their current clean entries
SELECT
d.id AS document_id,
d.jobId as job_id,
cte.id AS clean_entry_id
FROM
docs d
LEFT JOIN currentCleanEntries cte ON cte.documentId = d.id
WHERE cte.id is null or cte.id is not null and cte.fail is null
),
doc_text_entries AS (
-- Documents with their current text entries
SELECT
d.id AS document_id,
d.document_id,
cte.id AS text_entry_id
FROM
docs d
LEFT JOIN currentTextEntries cte ON cte.documentId = d.id
doc_clean_entries d
LEFT JOIN currentTextEntries cte ON cte.cleanEntryId = d.clean_entry_id
),
required_results AS (
-- All required document-query-version combinations
SELECT
d.id AS document_id,
d.document_id,
cqdt.queryId,
cqdt.queryVersion,
dte.text_entry_id
FROM
docs d
JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.jobId
JOIN doc_text_entries dte ON dte.document_id = d.id
doc_clean_entries d
JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.job_id
JOIN doc_text_entries dte ON dte.document_id = d.document_id
and dte.text_entry_id IS NOT NULL
),
existing_results AS (
@@ -152,26 +163,37 @@ SELECT (
// -- Get all documents for this job
// SELECT id, jobId FROM documents WHERE jobId = $1
// ),
// doc_clean_entries as (
// -- Documents with their current clean entries
// SELECT
// d.id AS document_id,
// d.jobId as job_id,
// cte.id AS clean_entry_id
// FROM
// docs d
// LEFT JOIN currentCleanEntries cte ON cte.documentId = d.id
// WHERE cte.id is null or cte.id is not null and cte.fail is null
// ),
// doc_text_entries AS (
// -- Documents with their current text entries
// SELECT
// d.id AS document_id,
// d.document_id,
// cte.id AS text_entry_id
// FROM
// docs d
// LEFT JOIN currentTextEntries cte ON cte.documentId = d.id
// doc_clean_entries d
// LEFT JOIN currentTextEntries cte ON cte.cleanEntryId = d.clean_entry_id
// ),
// required_results AS (
// -- All required document-query-version combinations
// SELECT
// d.id AS document_id,
// d.document_id,
// cqdt.queryId,
// cqdt.queryVersion,
// dte.text_entry_id
// FROM
// docs d
// JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.jobId
// JOIN doc_text_entries dte ON dte.document_id = d.id
// doc_clean_entries d
// JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.job_id
// JOIN doc_text_entries dte ON dte.document_id = d.document_id
// and dte.text_entry_id IS NOT NULL
// ),
// existing_results AS (
internal/database/repository/job_test.go
+39 -4
View File
@@ -231,6 +231,32 @@ func TestJobSync(t *testing.T) {
assert.NoError(t, err)
assert.True(t, isSynced)
documentNoCleanID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Jobid: jobId,
Hash: "example_noclean",
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
bucket := "example_bucket"
key := "example_key"
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentNoCleanID,
Version: 1,
Fail: repository.NullCleanfailtype{
Valid: true,
Cleanfailtype: repository.CleanfailtypeInvalidMimetype,
},
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Jobid: jobId,
Hash: "example_hash",
@@ -241,11 +267,16 @@ func TestJobSync(t *testing.T) {
assert.NoError(t, err)
assert.False(t, isSynced)
keytwo := "example_key_2"
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
Bucket: &bucket,
Key: &keytwo,
Mimetype: repository.NullCleanmimetypes{
Valid: true,
Cleanmimetypes: repository.CleanmimetypesApplicationPdf,
},
})
assert.NoError(t, err)
@@ -403,8 +434,12 @@ func TestJobSync(t *testing.T) {
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
Bucket: &bucket,
Key: &key,
Mimetype: repository.NullCleanmimetypes{
Valid: true,
Cleanmimetypes: repository.CleanmimetypesApplicationPdf,
},
})
assert.NoError(t, err)
internal/database/repository/models.go
+112 -10
View File
@@ -11,6 +11,104 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
type Cleanfailtype string
const (
CleanfailtypeInvalidMimetype Cleanfailtype = "invalid_mimetype"
)
func (e *Cleanfailtype) Scan(src interface{}) error {
switch s := src.(type) {
case []byte:
*e = Cleanfailtype(s)
case string:
*e = Cleanfailtype(s)
default:
return fmt.Errorf("unsupported scan type for Cleanfailtype: %T", src)
}
return nil
}
type NullCleanfailtype struct {
Cleanfailtype Cleanfailtype
Valid bool // Valid is true if Cleanfailtype is not NULL
}
// Scan implements the Scanner interface.
func (ns *NullCleanfailtype) Scan(value interface{}) error {
if value == nil {
ns.Cleanfailtype, ns.Valid = "", false
return nil
}
ns.Valid = true
return ns.Cleanfailtype.Scan(value)
}
// Value implements the driver Valuer interface.
func (ns NullCleanfailtype) Value() (driver.Value, error) {
if !ns.Valid {
return nil, nil
}
return string(ns.Cleanfailtype), nil
}
func (e Cleanfailtype) Valid() bool {
switch e {
case CleanfailtypeInvalidMimetype:
return true
}
return false
}
type Cleanmimetypes string
const (
CleanmimetypesApplicationPdf Cleanmimetypes = "application/pdf"
)
func (e *Cleanmimetypes) Scan(src interface{}) error {
switch s := src.(type) {
case []byte:
*e = Cleanmimetypes(s)
case string:
*e = Cleanmimetypes(s)
default:
return fmt.Errorf("unsupported scan type for Cleanmimetypes: %T", src)
}
return nil
}
type NullCleanmimetypes struct {
Cleanmimetypes Cleanmimetypes
Valid bool // Valid is true if Cleanmimetypes is not NULL
}
// Scan implements the Scanner interface.
func (ns *NullCleanmimetypes) Scan(value interface{}) error {
if value == nil {
ns.Cleanmimetypes, ns.Valid = "", false
return nil
}
ns.Valid = true
return ns.Cleanmimetypes.Scan(value)
}
// Value implements the driver Valuer interface.
func (ns NullCleanmimetypes) Value() (driver.Value, error) {
if !ns.Valid {
return nil, nil
}
return string(ns.Cleanmimetypes), nil
}
func (e Cleanmimetypes) Valid() bool {
switch e {
case CleanmimetypesApplicationPdf:
return true
}
return false
}
type Querytype string
const (
@@ -129,11 +227,13 @@ type Collectorversion struct {
}
type Currentcleanentry struct {
ID pgtype.UUID `db:"id"`
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
ID pgtype.UUID `db:"id"`
Documentid pgtype.UUID `db:"documentid"`
Bucket *string `db:"bucket"`
Key *string `db:"key"`
Version int32 `db:"version"`
Mimetype NullCleanmimetypes `db:"mimetype"`
Fail NullCleanfailtype `db:"fail"`
}
type Currentclientcansync struct {
@@ -183,11 +283,13 @@ type Document struct {
}
type Documentclean struct {
ID pgtype.UUID `db:"id"`
Documentid pgtype.UUID `db:"documentid"`
Version int32 `db:"version"`
Bucket string `db:"bucket"`
Key string `db:"key"`
ID pgtype.UUID `db:"id"`
Documentid pgtype.UUID `db:"documentid"`
Version int32 `db:"version"`
Bucket *string `db:"bucket"`
Key *string `db:"key"`
Mimetype NullCleanmimetypes `db:"mimetype"`
Fail NullCleanfailtype `db:"fail"`
}
type Documententry struct {
internal/database/repository/result_test.go
+30 -8
View File
@@ -71,11 +71,17 @@ func TestResults(t *testing.T) {
assert.NoError(t, err)
assert.False(t, issynced)
bucket := "example_bucket"
key := "example_key"
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
Bucket: &bucket,
Key: &key,
Mimetype: repository.NullCleanmimetypes{
Valid: true,
Cleanmimetypes: repository.CleanmimetypesApplicationPdf,
},
})
assert.NoError(t, err)
@@ -227,11 +233,17 @@ func TestResultValues(t *testing.T) {
})
assert.NoError(t, err)
bucket := "example_bucket"
key := "example_key"
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
Bucket: &bucket,
Key: &key,
Mimetype: repository.NullCleanmimetypes{
Valid: true,
Cleanmimetypes: repository.CleanmimetypesApplicationPdf,
},
})
assert.NoError(t, err)
cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID)
@@ -419,11 +431,17 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
assert.NoError(t, err)
assert.Len(t, qs, 0)
bucket := "example_bucket"
key := "example_key"
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
Bucket: &bucket,
Key: &key,
Mimetype: repository.NullCleanmimetypes{
Valid: true,
Cleanmimetypes: repository.CleanmimetypesApplicationPdf,
},
})
assert.NoError(t, err)
@@ -490,8 +508,12 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentTwoID,
Version: 1,
Bucket: "hi",
Key: "hello",
Bucket: &bucket,
Key: &key,
Mimetype: repository.NullCleanmimetypes{
Valid: true,
Cleanmimetypes: repository.CleanmimetypesApplicationPdf,
},
})
assert.NoError(t, err)
internal/database/repository/text_test.go
+6 -2
View File
@@ -48,8 +48,12 @@ func TestTextExtraction(t *testing.T) {
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: id,
Version: 1,
Bucket: bucket,
Key: key,
Bucket: &bucket,
Key: &key,
Mimetype: repository.NullCleanmimetypes{
Valid: true,
Cleanmimetypes: repository.CleanmimetypesApplicationPdf,
},
})
assert.NoError(t, err)