jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/splitqueryrunning (pull request #57)

Browse Source 
Split Query Running + Debugging Full Flow

* completedquerysyncrunner

* spliitinglogic

* synccomplete

* informdependents

* only push same collector

* deps

* livetesting

* foundissue

* some issues resolved

* activeupdate

* collectorupdatefixes

* fix dbquesries

* tests

* tests

* pollingdebug
This commit is contained in:
Michael McGuinness
2025-02-11 15:22:59 +00:00
parent 24a038ec3d
commit 71f9802e1a
108 changed files with 3013 additions and 1666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/repository/text.sql.go
+21 -15
View File
@@ -36,12 +36,12 @@ func (q *Queries) AddDocumentTextEntry(ctx context.Context, arg *AddDocumentText
}
const getDocumentTextEntry = `-- name: GetDocumentTextEntry :one
SELECT dc.documentId, dc.bucket, dc.key
SELECT dc.documentId, dc.bucket, dc.key, dc.version
FROM documentTextExtractions AS dc
JOIN documents AS d ON d.id = dc.documentId
JOIN collectors as c ON d.jobId = c.jobId
JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1)
ORDER BY d.id DESC LIMIT 1
`
@@ -49,21 +49,27 @@ type GetDocumentTextEntryRow struct {
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
}
// GetDocumentTextEntry
//
// SELECT dc.documentId, dc.bucket, dc.key
// SELECT dc.documentId, dc.bucket, dc.key, dc.version
// FROM documentTextExtractions AS dc
// JOIN documents AS d ON d.id = dc.documentId
// JOIN collectors as c ON d.jobId = c.jobId
// JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1)
// ORDER BY d.id DESC LIMIT 1
func (q *Queries) GetDocumentTextEntry(ctx context.Context, documentid pgtype.UUID) (*GetDocumentTextEntryRow, error) {
row := q.db.QueryRow(ctx, getDocumentTextEntry, documentid)
var i GetDocumentTextEntryRow
err := row.Scan(&i.Documentid, &i.Bucket, &i.Key)
err := row.Scan(
&i.Documentid,
&i.Bucket,
&i.Key,
&i.Version,
)
return &i, err
}
@@ -72,9 +78,9 @@ SELECT EXISTS(
SELECT 1
FROM documentTextExtractions AS dc
JOIN documents AS d ON d.id = dc.documentId
JOIN collectors as c ON d.jobId = c.jobId
JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1)
)
`
@@ -84,9 +90,9 @@ SELECT EXISTS(
// SELECT 1
// FROM documentTextExtractions AS dc
// JOIN documents AS d ON d.id = dc.documentId
// JOIN collectors as c ON d.jobId = c.jobId
// JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1)
// )
func (q *Queries) IsDocumentTextExtracted(ctx context.Context, documentid pgtype.UUID) (bool, error) {
row := q.db.QueryRow(ctx, isDocumentTextExtracted, documentid)