jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/splitqueryrunning (pull request #57)

Browse Source 
Split Query Running + Debugging Full Flow

* completedquerysyncrunner

* spliitinglogic

* synccomplete

* informdependents

* only push same collector

* deps

* livetesting

* foundissue

* some issues resolved

* activeupdate

* collectorupdatefixes

* fix dbquesries

* tests

* tests

* pollingdebug
This commit is contained in:
Michael McGuinness
2025-02-11 15:22:59 +00:00
parent 24a038ec3d
commit 71f9802e1a
108 changed files with 3013 additions and 1666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/repository/clean.sql.go
+21 -15
View File
@@ -36,12 +36,12 @@ func (q *Queries) AddDocumentCleanEntry(ctx context.Context, arg *AddDocumentCle
}
const getDocumentCleanEntry = `-- name: GetDocumentCleanEntry :one
SELECT dc.documentId, dc.bucket, dc.key
SELECT dc.documentId, dc.bucket, dc.key, dc.version
FROM documentCleans AS dc
JOIN documents AS d ON d.id = dc.documentId
JOIN collectors as c ON d.jobId = c.jobId
JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1)
ORDER BY d.id DESC LIMIT 1
`
@@ -49,21 +49,27 @@ type GetDocumentCleanEntryRow struct {
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
}
// GetDocumentCleanEntry
//
// SELECT dc.documentId, dc.bucket, dc.key
// SELECT dc.documentId, dc.bucket, dc.key, dc.version
// FROM documentCleans AS dc
// JOIN documents AS d ON d.id = dc.documentId
// JOIN collectors as c ON d.jobId = c.jobId
// JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1)
// ORDER BY d.id DESC LIMIT 1
func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.UUID) (*GetDocumentCleanEntryRow, error) {
row := q.db.QueryRow(ctx, getDocumentCleanEntry, documentid)
var i GetDocumentCleanEntryRow
err := row.Scan(&i.Documentid, &i.Bucket, &i.Key)
err := row.Scan(
&i.Documentid,
&i.Bucket,
&i.Key,
&i.Version,
)
return &i, err
}
@@ -72,9 +78,9 @@ SELECT EXISTS(
SELECT 1
FROM documentCleans AS dc
JOIN documents AS d ON d.id = dc.documentId
JOIN collectors as c ON d.jobId = c.jobId
JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1)
)
`
@@ -84,9 +90,9 @@ SELECT EXISTS(
// SELECT 1
// FROM documentCleans AS dc
// JOIN documents AS d ON d.id = dc.documentId
// JOIN collectors as c ON d.jobId = c.jobId
// JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1)
// )
func (q *Queries) IsDocumentClean(ctx context.Context, documentid pgtype.UUID) (bool, error) {
row := q.db.QueryRow(ctx, isDocumentClean, documentid)