jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/jobsynced (pull request #72)

Browse Source 
Job Status Get and DB tidy up

* initalquery

* tests

* shorttests

* testing queries

* job

* solvedthequery

* updatingdb

* fixingtests

* repotests

* shorttests

* docker

* testspassed
This commit is contained in:
Michael McGuinness
2025-02-20 19:02:44 +00:00
parent 0ea544926b
commit 3d434eedb8
74 changed files with 2395 additions and 1335 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+3 -1
View File
@@ -32,6 +32,8 @@
LICENSE
README.md
**/*_test.go
cmd/*_test/
.bitbucket/
.devbox/
build/
@@ -55,4 +57,4 @@ sqlc.yml
vaccum.conf.yaml
.*.yml
**/*.cast
**/*.http
**/*.http
api/docTextRunner/runner_test.go
+4 -3
View File
@@ -70,11 +70,12 @@ func TestDocCleanRunner(t *testing.T) {
pgxmock.NewRows([]string{"isextracted"}).
AddRow(false),
)
cleanId := database.MustToDBUUID(uuid.New())
pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(doc.ID)).WillReturnRows(
pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}).
AddRow(database.MustToDBUUID(doc.ID), inloc.Bucket, inloc.Key, int32(1)),
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version"}).
AddRow(cleanId, database.MustToDBUUID(doc.ID), inloc.Bucket, inloc.Key, int32(1)),
)
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(database.MustToDBUUID(doc.ID), int32(1), inloc.Bucket, inloc.Key).
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(int32(1), inloc.Bucket, inloc.Key, cleanId).
WillReturnResult(pgxmock.NewResult("", 1))
mockSQS.EXPECT().
api/queryRunner/runner_test.go
+26 -15
View File
@@ -21,7 +21,6 @@ import (
"github.com/aws/aws-sdk-go-v2/service/sqs/types"
"github.com/go-playground/validator/v10"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
@@ -81,15 +80,12 @@ func TestQueryRunner(t *testing.T) {
DocumentID: doc.DocumentID,
}
pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}).
AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)),
)
cleanEntryId := uuid.New()
textEntryId := uuid.New()
pool.ExpectQuery("name: GetDocumentTextEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}).
AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)),
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "cleanEntryId"}).
AddRow(database.MustToDBUUID(textEntryId), database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1), database.MustToDBUUID(cleanEntryId)),
)
pool.ExpectQuery("name: GetQuery :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows(
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
@@ -99,17 +95,32 @@ func TestQueryRunner(t *testing.T) {
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)),
)
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version).
requiredResultId := uuid.New()
strVal := "{\"examplekey\":\"example_value\"}"
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"queryId", "type", "value"}).
AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, "{\"examplekey\":\"example_value\"}"),
pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
AddRow(database.MustToDBUUID(requiredResultId), database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, &strVal),
)
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(qcfg)),
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows(
pgxmock.NewRows([]string{"config"}).
AddRow([]byte(qcfg)),
)
pool.ExpectExec("name: SetResult :exec").WithArgs(database.MustToDBUUID(query.ID), database.MustToDBUUID(params.DocumentID), pgxmock.AnyArg(), int32(1), int32(1), query.Version).
pool.ExpectBegin()
resultId := uuid.New()
pool.ExpectQuery("name: AddResult :one").WithArgs(database.MustToDBUUID(query.ID), pgxmock.AnyArg(), database.MustToDBUUID(textEntryId), query.Version).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(resultId)),
)
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
AddRow(database.MustToDBUUID(requiredResultId), database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, &strVal),
)
pool.ExpectExec("name: AddResultDependency :exec").WithArgs(database.MustToDBUUID(resultId), database.MustToDBUUID(requiredResultId)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(database.MustToDBUUID(query.ID), database.MustToDBUUID(doc.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"queryId"}).
api/queryService/api.gen.go
+54 -41
View File
@@ -38,6 +38,12 @@ const (
RightClosedInterval FieldFilterCondition = "right_closed_interval"
)
// Defines values for JobStatus.
const (
INSYNC JobStatus = "IN_SYNC"
NOTSYNCED JobStatus = "NOT_SYNCED"
)
// Defines values for QueryType.
const (
CONTEXTFULL QueryType = "CONTEXT_FULL"
@@ -123,6 +129,9 @@ type Job struct {
// Id The job id
Id openapi_types.UUID `json:"id"`
// Status Specifies the status of a job.
Status JobStatus `json:"status"`
}
// JobClient defines model for JobClient.
@@ -188,6 +197,9 @@ type JobCreate struct {
ClientId openapi_types.UUID `json:"client_id"`
}
// JobStatus Specifies the status of a job.
type JobStatus string
// JobUpdate defines model for JobUpdate.
type JobUpdate struct {
// CanSync Specifies whether the job is actively syncing
@@ -598,47 +610,48 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL
// Base64 encoded, gzipped, json marshaled Swagger object
var swaggerSpec = []string{
"H4sIAAAAAAAC/+RbX2/bthb/KoTufTTsdN2T37o0HRz0rr1tejFgCAyKOraZ0aJCUk69wN/9gjzUP0u0",
"5CTaMuypTUSeQ/7O7/whD/MYMbnNZAqp0dH8MdJsA1vq/nspOKTmUgE1YH/OlMxAGQ7ua0q37rcJaKZ4",
"ZrhMo3l0swHC3DziBkwis88gmkfaKJ6uo8NhEim4z7mCJJr/hlJuy1EyvgNmosPEK/+WJZ3KGU2Xep+y",
"9gIWK2KqNXBNqBDygadrQpnhOyB2mq7WFUspgKZW5dN31Fr91fdMKvMeDOVCt2V+pnshaUJWUhFwQ4lR",
"fL0GRRToTKYaptHkaM93Ml7ypHuBdzImPCEKBHWbNNKhgLKtqJVUW2qieZTnPGnvYRLJ3GS5WQrJKMrt",
"UlN8JTwlDxvONjUt5A+ekRUXQB64ECQGspJ5mljl8J1uM+H0vZ3PZo9xzn4Hc5g9Iq48Ocwe72Ts/jV8",
"C9rQbXZYPqJgnhymf/Csa9HaUJM7cP6tYBXNo3/NKjbPPJVnaIyvOPaYgB7VUtZt0JpfS2VtYDKpNY9F",
"iYW1hxUI2u0/zbdWl12bAANWHU+XmZJrBdqScUW5gKSmvNojKr9BepymkueQI3taM36TRysOIlmuuDCg",
"OrbzwX1whtVMZkBiqiEhMiVuIkGikB0VOe6OG9j22uCDnYuio8phqFJ0b3/m6Rq0XcBT1lVOJhlVdAt2",
"fnvbkCbLIph0MJtqQ+xnIleVwGmAdMoERXFNVlwNFNYVOga4ucQYd5+D2tuAkQujrcvHBf8gGeDz3Y7Q",
"xf+68QJ2cZwjTIp8m7agZzJNeBFVBrLkspxzmHjKhuOz/VKggizlqYUnbESkb7c0/EZyDatcFKEUedng",
"e0tok9VH+Nb2MKkhUi6lB/jLOoYnAhBuf1UapdTUiEQCtF6aDbX61y7Bq+JHJqSGZMlTA2pHRTSJZAZp",
"/WcBK7NsD1N8ven6PU+ZyBNweQD/1xXmFsl/QGu67kj2Xd7wLeX3ORCe2Pyx4qAwl6aGm/353A/w/lrG",
"55QeXzNgdi2aPGzAbHywcj6rffUh9q7+sGvoqkAwHwbdvyhqklJwDEKma+v8Q3L86bDyFNTqS55U0ATA",
"xJruT6rmekEcgtjLFLlOduH5fRhJIYAZ2RFp61/JViYg2lkOYVnuQOlgrPDQ+TFF3GSF6Ib78NS8/aHa",
"oHXrNaZwF2gCERS/2SBcELUhfVDFUN+ti4NddcOpZLl4X2yNai0ZpwYSu5ZB5bCwtZs5jSOOeS6OW57y",
"bb5dMktcUKdV+sGlTlf1bYAkkuVby0cv5UzdBr6bpym2M23ZpQfqDFXfnk1hOAKLbVlqcuwCfX6G3ArT",
"mFQOVu66xeihR2NfPKNgO6yzOHGFXZDXWPa5JEANHrYymeUWB7c4LAjPT4I+QJXK+4CrzubhE0lux2Bt",
"WJ/6j49b/zin72RS4GZp1CroiPSVrgDdz7+Bem4Z2IXWR67Nf3NQXnlzKffVh+ZK7CzrJn7AYApbTfve",
"A02htgs4lNBa6dO93EWlgQRnMl3xdVv6pft9rvAKqyBzKXlQudx97uhe4cvXFuegUJhq2UuPYqTPLIv3",
"Tab07uc4uOHPAyh2Ywd2FstORCuVt6ALUi8YWs6iRgoPJ+jxdwXYzQ4idwPafAF3d9KGr8g4nbG5qrfL",
"xGQkcTSma8pTPewiGuuPoHP8r8MjrJ5cA9oNtC04nlKM1nd3vI4evPDOvg2Yu9tpb+KLK9GO9gAI0Ol8",
"hQLDq/HMCGUkl733GbQCSnExdP310y/Lq19vvry7vPn0JZpEl59+ubn69Wb54dvHj50XN05vKEv2BXyc",
"lxwF/WfG+UIoOzPe9zt0IVmM4tjtxH9wV+Mr2V7Ju8+L0s2a+3P2IJ8U24A2fu8a1I4zrAAMN64V0zXu",
"3edFNIlKa0VvphfTC9cdyiClGY/m0dvpxfStLd+p2bhNzlh1qyMxaBzFVheLNaEunvrq7YEbbB5lSu54",
"AglJsFk2xQtHXNAiKef7uyO0Emjzk0z2PqIbr55mmeDYoJrdaaQbhsO+YNnodB6aPmdUDu4X6ONuzz9c",
"vHkx3dXFp1N8hB2ixdzKEqJzxkDrVS7Efmrt8uPFRUcgTndUuG6gQ4rEMrGjD5NI59sttYWZB7VhE8sN",
"utY2CqDar8ia6NZO9WaePfLkYFWuwXTFNaM47JyxNQYdVlg83hNuNFm8b1v4ZzA18zZwvngxnKv7xzDO",
"JQeHQltvOLk5P3bw398USlN0RJum+BkMoTWUFu9PWMI6XqEymv92+t7LyzSSKG8YF57sQOu+xaXkHGuu",
"JuUnNVj7jlS3LhywTShgamxHcpeZGxHAsi/o+Dj5T3B8n70GOf5F0MC5Tw7PctKBHDqiEG6gZFGPJ9/5",
"tsaAaG1PrWeH6msZj2Su6tLgNQXpaxmPHaHvHKSFUa9l3LLoDDu/YcMuUm64M23tzUampF1u25L+vQE+",
"PhjJms2XDa/JoleNNzkj+bSlTTApeFSqdxw18/vnKC0GnJ+a3eXZybxc+PJ4STnkUaPk4pOYYyL2mDSy",
"cMPhBqRgKwMB/auTbRnAB2TaUeP2OTk2YLgxE+xJYpTpdUAYtk44Y/Um7nnuWE7td8yyzfDT/lrGi2Ts",
"6rncVCgHlktv+m4A7mr4AI9swHLsm+XKnuCk5bm52el5BW5bbfkMB+7kxDiF2FEf8DmeXW11bB8fQLq6",
"t1fjexnXCABVMdbp/ZcbYL9jIcZypWxp716L2oNb4+Vm08rVO1QY09ebj5fD5dGGahIDpKR81xrG/qp6",
"FxsE3qFSVKYOj+mJquesk3CF6AiebS1/XzS8eqO9aDbl6s9r3QNXprgBxWnb/PUm4Ijmr6vpMP674x28",
"bIWG15JBjriWyn0JQkEON6vBjQGHW7y0Pft4i73NceJqvXn1mg5EaJWRD7n3HtiAUUs3e8JBB019uqKq",
"7DqSY/m2egjdUQ48Pe6EBVaJTqOwavvU8HhbNln+6kKq5uMDCqjRXfuZN41ozTFrpB6+lLXRWb46M0U3",
"+dUwqDM9XH0HlmN+cG1rlbs+Mw1miubfmxzdoIE2o/Op3q0fTqqX1u+73x2R7cbB6B8kPidROEH9tLNz",
"QO26CfZZySRnZUMUbE2fKxHNo40xmZ7PZjTjU//HalMmt7Pdm8iyxWs7lvepsLfGP7yDhBjpr+B1xdbm",
"FfxhMkzMnYxrMmpXDWcIqE4vTVGt08tQmVXh5YU18B8qBYvxmpRmaX+4Pfw/AAD///6KTOeVOgAA",
"H4sIAAAAAAAC/+RbW3PbNhb+KxjuPmokt+mT3lLb6ciTjbOxstOdTkYDgkcSXJCgAVCO6tF/38GFNxEQ",
"KdtsvdOnxCZwDvCd71yAAz9FhKc5zyBTMpo/RZJsIcXmv5eMQqYuBWAF+udc8ByEomC+Zjg1v01AEkFz",
"RXkWzaPlFhAx85AZMInUPodoHkklaLaJDodJJOChoAKSaP6blfKtGsXjeyAqOkyc8q954lVOcLaS+4x0",
"F7BYI1WvgUqEGeOPNNsgTBTdAdLTZL2umHMGONMqn7+jzuqvv+dcqCtQmDLZlfkZ7xnHCVpzgcAMRUrQ",
"zQYEEiBznkmYRpOjPd/zeEUT/wLveYxoggQwbDapuEHBytai1lykWEXzqCho0t3DJOKFygu1YpxgK9en",
"pvyKaIYet5RsG1rQHzRHa8oAPVLGUAxozYss0crhO05zZvS9m89mT3FBfgd1mD1ZXGlymD3d89j8q2gK",
"UuE0P6yerGCaHKZ/0Ny3aKmwKgw4/xSwjubRP2Y1m2eOyjNrjDs79piADtVK1regNe8qZV1gci4ljVmF",
"hbaHFgjS7D8rUq1Lr42BAq2OZqtc8I0Aqcm4xpRB0lBe79EqX1p6nKaS45Ahe9YwfptHawosWa0pUyA8",
"2/lgPhjDSsJzQDGWkCCeITMRWaKgHWaF3R1VkPba4IOea0VHtcNgIfBe/0yzDUi9gOesq5qMcixwCnp+",
"d9uQJasymHiYjaVC+jPi61rgNEA6oYKiqERrKgYK84WOAW7ObYx7KEDsdcAomJLa5eOSf5AM8Hm/I/j4",
"3zRewC6Gc4hwVqRZB3rCs4SWUWUgSy6rOYeJo2w4PusvJSqWpTTT8ISNaOnrl2a/oULCumBlKLW8bPG9",
"I7TN6iN8G3uYNBCpltID/GUTwxMByG5/XRml0tSKRAykXKkt1vo3JsGL8kfCuIRkRTMFYodZNIl4Dlnz",
"ZwZrteoOE3Sz9f2eZoQVCZg8YP/nC3OL5F8gJd54kr3PG75m9KEARBOdP9YUhM2lmaJqfz73A7y/4fE5",
"pcddDkSvRaLHLaitC1bGZ6WrPtje1B96Db4KxObDoPuXRU1SCY6B8WyjnX9Ijj8dVoZIGJZwb3gcyLZG",
"ar3LSY3myQR8w2NbD/5JlWCvAYZg9ToFspFdRo1yryGMOGNAFPdE6eZXlPIEWDdDWlhWOxAyGGccdG5M",
"GXNJKbrlejRT736sN6hDwsamfxOkAtHXftMBvCR5S/qgaqO5WxNDfTXHqUS7uCq3hqXkhGIFiV7LoFKa",
"6bpPncbRjnkpjinNaFqkK6KJC+K0Sje40mkqxi2ghJMi1Xx0Us7UreC7ep5iPVOXbHKgzlDl7tgUhiOw",
"2I6lJscu0OdnllthGqPawapddxg99FjtCm8rWA/zFjamKAzy2paMJoFgZQ9qOc8LjYNZnC0mz0+gLkBV",
"yvuAq8/14dNMocfYurI59W8ft/52Tu9lUuBWatQK6oj0ta4A3UM3BnWhaE6zZpSmJC5TTFmmLz6t7v77",
"6TKaRJ9ul+a/11fe6vmGx+dflb20XvWZ5iOV6t8FCKe8vZSH+kN7JXqWBsANGOwvWtO+9+RVqvVZyUro",
"rPT5IcWEwIHeRHi2ppuu9Evz+0LYu7bScyrJg+p6/wHJv8LXL2TOQaE01aqXHuVIl8YWV22m9O7nOJLa",
"nwdQbKkHeitzI6JTN3SgC1IvGMfOokYGjyfo8f8KsJkdRG4JUn0Bc8nTha9Mb95EUBf3VRZUHBka4w2m",
"mRx2Y26LnaBz/MfjEVpPIcHaDaSubp5T+TZ3d7yOHrxsc6ELmLmE6m7ii6kHj/YAFqDTydEKDK/GMeNU",
"YtQzOwGlTI03d7efVte/Lr+8v1zefokm0eXtp+X1r8vVh68fP3pzpNEbypJ9Ad/OS46C/gvjfCmUnBnv",
"+x26lMxGcexu4j+YO/w1767k/edF5Wbt/Rl7oFtBtiCV27sEsaPEVgCKKtMz8o17/3kRTaLKWtEP04vp",
"hWlj5ZDhnEbz6N30YvpOnxWw2ppNzkh9hcRt0DiKrSYWS4RNPHWl4iNVtsuVC76jCSQosV29qb0ZtQta",
"JNV8d1FlrQRS/cyTvYvoyqnHec6o7aTN7qWlmw2HfcGy1ZI9tH1OiQLML6yPmz3/ePHDq+mub2iN4iPs",
"LFrErCxBsiAEpFwXjO2n2i4/XVx4AnG2w8y0LQ1SKOaJHn2YRLJIU6wLMwdqyyaaG3gjdRSwau8sa6Jv",
"eqoz8+yJJgetcgPKF9eUoLAzxpY26JDS4vEeUSXR4qpr4V9ANczbwvni1XCuLzvDOFccHAptszNm5vzk",
"4b+7luSqbN22TfELKIQbKC2uTlhCO16pMpr/dvqSzclUHAlnGBOe9EDtvuUN6NzWXG3KTxqw9p3fvplw",
"QLahgClt35SazNyKAJp9Qce3k/8Ex3fZa5DjXwQNXLjk8CInHcihIwrZDVQs6vHke9d/GRCt9an17FB9",
"w+ORzFXfULylIH3D47Ej9L2BtDTqDY87Fp3ZFnXYsIuMKmpM23hckguul9u1pHsYYV9JjGTN9hOMt2TR",
"69bjoZF8WtMmmBQcKvWDk4b53buZDgPOT83mpu5kXi59ebykHPKoUXLxScxtInaYtLJwy+EGpGAtwwL6",
"VyfbKoAPyLSjxu1zcmzAcGMm2JPEqNLrgDCsnXBGmh3j89yxmtrvmFVP4+f9DY8XydjVc7WpUA6slt72",
"3QDc9fABHtmC5dg3q5U9w0mrc3O7rfQG3Lbe8hkO7OXEOIXYUdPxJZ5db3VsHx9Auqa31+N7GdcKAHUx",
"5vX+yy2Q320hRgohdGlvnrWablXziWnbyvWDWRjT19uvrMPl0RZLFANkqHqAG8b+un7AGwTeoFJWpgaP",
"6Ymq56yTcI3oCJ6tLf9QNrx6oz1rN+Wa74DNS1wiqAJBcdf8zSbgiOZvqvEY//3xDl63QrPXkkGOmJbK",
"QwVCSQ4zq8WNAYdbe2l79vHW9jbHiavN5tVbOhBZq4x8yH1wwAaMWrnZMw461tSnK6rariM5lmurh9Ad",
"5cDT4062wKrQaRVWXZ8aHm+rJstfXUg1fHxAATW6a7/wptFac8waqYcvVW10lq/OVNlNfjMM8qaH6+9A",
"CpsfTNtaFJl9PxTKFO0/jDm6QQOpRudTs1s/nFSvrd91vz2RbWlgdK8fX5IojKB+2uk5IHZ+gn0WPClI",
"1RAFXdMXgkXzaKtULuezGc7p1P1V3ZTwdLb7IdJscdqO5d2W9pb2LwQhQYq7K3hZs7V9BX+YDBNzz+OG",
"jMZVwxkC6tNLW1Tn9DJUZl14OWEt/IdKscV4Q0q7tD98O/wvAAD//8+zA+A+OwAA",
}
// GetSwagger returns the content of the embedded swagger specification file
api/queryService/job.go
+5 -4
View File
@@ -26,15 +26,16 @@ func (s *Controllers) CreateJob(ctx echo.Context) error {
}
func (s *Controllers) GetJob(ctx echo.Context, id types.UUID) error {
job, err := s.svc.Job.Get(ctx.Request().Context(), id)
job, err := s.svc.Job.GetWithStatus(ctx.Request().Context(), id)
if err != nil {
return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Unable to find job: %s", err))
}
return ctx.JSON(http.StatusOK, Job{
Id: job.ID,
ClientId: job.ClientID,
CanSync: job.CanSync,
Id: job.Job.ID,
ClientId: job.Job.ClientID,
CanSync: job.Job.CanSync,
Status: parseJobStatus(job.Status),
})
}
api/queryService/job_test.go
+7 -7
View File
@@ -57,22 +57,17 @@ func TestCreateJob(t *testing.T) {
ctx := e.NewContext(req, rec)
id := uuid.New()
collId := uuid.New()
pool.ExpectQuery("name: CreateJob :one").WithArgs(database.MustToDBUUID(body.ClientId)).WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(id)),
)
pool.ExpectBeginTx(pgx.TxOptions{})
pool.ExpectQuery("name: CreateCollector :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(collId)),
)
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(collId)).WillReturnRows(
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pgxmock.NewRows([]string{"version"}).
AddRow(int32(1)),
)
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(collId), int32(1)).
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
@@ -117,6 +112,10 @@ func TestGetJob(t *testing.T) {
pgxmock.NewRows([]string{"id", "name", "canSync"}).
AddRow(database.MustToDBUUID(j.ClientID), "client_name", true),
)
pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows(
pgxmock.NewRows([]string{"issynced"}).
AddRow(true),
)
err = cons.GetJob(ctx, j.ID)
assert.NoError(t, err)
@@ -129,6 +128,7 @@ func TestGetJob(t *testing.T) {
Id: j.ID,
ClientId: j.ClientID,
CanSync: j.CanSync,
Status: queryservice.INSYNC,
}, res)
}
api/queryService/jobcollector_test.go
+8 -10
View File
@@ -44,7 +44,6 @@ func TestUpdateJobCollector(t *testing.T) {
cfg.QueueClient = mockSQS
current := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
ActiveVersion: 1,
LatestVersion: 4,
@@ -84,8 +83,8 @@ func TestUpdateJobCollector(t *testing.T) {
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(current.JobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(current.ID), database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")),
pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")),
)
pool.ExpectQuery("name: AllQueriesExist :one").WithArgs([]pgtype.UUID{database.MustToDBUUID((*body.Fields)[0].QueryId)}).
WillReturnRows(
@@ -93,15 +92,15 @@ func TestUpdateJobCollector(t *testing.T) {
AddRow(true),
)
pool.ExpectBeginTx(pgx.TxOptions{})
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.ID)).WillReturnRows(
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.JobID)).WillReturnRows(
pgxmock.NewRows([]string{"version"}).
AddRow(int32(5)),
)
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ID), int32(2)).
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(2)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.ID), int32(5), *body.MinimumCleanerVersion).
pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(5), *body.MinimumCleanerVersion).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ID), "a", database.MustToDBUUID((*body.Fields)[0].QueryId), int32(5)).
pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.JobID), "a", database.MustToDBUUID((*body.Fields)[0].QueryId), int32(5)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
@@ -144,7 +143,6 @@ func TestGetJobCollectorByJobId(t *testing.T) {
})
coll := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
MinCleanVersion: int32(1),
MinTextVersion: int32(2),
@@ -152,8 +150,8 @@ func TestGetJobCollectorByJobId(t *testing.T) {
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(coll.JobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(coll.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")),
pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(coll.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")),
)
err = cons.GetJobCollectorByJobId(ctx, coll.JobID)
api/queryService/parse.go
+10
View File
@@ -2,6 +2,7 @@ package queryservice
import (
"errors"
"queryorchestration/internal/job"
"queryorchestration/internal/query"
resultprocessor "queryorchestration/internal/query/result/processor"
@@ -84,3 +85,12 @@ func parseStringToUUIDArray(sids *[]string) (*[]uuid.UUID, error) {
return nil, nil
}
func parseJobStatus(status job.Status) JobStatus {
switch status {
case job.IN_SYNC:
return INSYNC
}
return NOTSYNCED
}
api/queryService/parse_test.go
+12
View File
@@ -1,6 +1,7 @@
package queryservice
import (
"queryorchestration/internal/job"
"queryorchestration/internal/query"
resultprocessor "queryorchestration/internal/query/result/processor"
"testing"
@@ -124,3 +125,14 @@ func TestParseStringToUUIDArray(t *testing.T) {
assert.NoError(t, err)
assert.Nil(t, out)
}
func TestParseSpecJobStatus(t *testing.T) {
qt := parseJobStatus(job.IN_SYNC)
assert.Equal(t, INSYNC, qt)
qt = parseJobStatus(job.NOT_SYNCED)
assert.Equal(t, NOTSYNCED, qt)
qt = parseJobStatus("invalid")
assert.Equal(t, NOTSYNCED, qt)
}
api/queryService/query_test.go
+8 -27
View File
@@ -11,7 +11,6 @@ import (
"queryorchestration/internal/document"
cleanversion "queryorchestration/internal/document/clean/version"
textversion "queryorchestration/internal/document/text/version"
"queryorchestration/internal/job"
"queryorchestration/internal/job/collector"
"queryorchestration/internal/query"
"queryorchestration/internal/query/result"
@@ -255,17 +254,8 @@ func TestTestQuery(t *testing.T) {
}),
})
j := job.Job{
ID: uuid.New(),
}
coll := collector.Collector{
ID: uuid.New(),
JobID: j.ID,
}
doc := document.Document{
ID: uuid.New(),
JobID: coll.JobID,
Hash: "example_hash",
ID: uuid.New(),
}
params := &result.Process{
QueryID: uuid.New(),
@@ -286,28 +276,19 @@ func TestTestQuery(t *testing.T) {
ctx := e.NewContext(req, rec)
reqID := database.MustToDBUUID(uuid.New())
pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "hash"}).
AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash),
)
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(doc.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")),
)
pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), &params.QueryVersion).WillReturnRows(
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
AddRow(database.MustToDBUUID(params.QueryID), repository.QuerytypeJsonExtractor, int32(1), params.QueryVersion+1, []byte("{\"path\":\"oldkey\"}"), []pgtype.UUID{reqID}),
)
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(params.QueryID), params.QueryVersion).
strVal := "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}"
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.QueryID), &params.QueryVersion, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"queryId", "type", "value"}).
AddRow(reqID, repository.QuerytypeContextFull, "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}"),
pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
AddRow(database.MustToDBUUID(uuid.New()), reqID, repository.QuerytypeContextFull, &strVal),
)
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion).WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte("{\"path\":\"oldkey\"}")),
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID)).WillReturnRows(
pgxmock.NewRows([]string{"config"}).
AddRow([]byte("{\"path\":\"oldkey\"}")),
)
err = cons.TestQuery(ctx, params.QueryID)
build/Dockerfile
+1 -5
View File
@@ -16,11 +16,7 @@ ARG TARGETARCH
SHELL ["/bin/bash", "-c"]
RUN --mount=type=cache,target=/go/pkg/mod/ \
--mount=type=bind,target=. \
while read -r dir; do \
CGO_ENABLED=0 GOARCH=$TARGETARCH go build -o "/bin/$(basename "$dir")" "$dir"; \
done < <(find ./cmd -type d -not -name "*_test" -mindepth 1 -maxdepth 1)
CGO_ENABLED=0 GOARCH=$TARGETARCH go build -o /bin ./cmd/...
FROM scratch AS final
database/migrations/00000000000005_collectors.up.sql
+25 -32
View File
@@ -1,16 +1,9 @@
CREATE TABLE collectors (
id uuid primary key DEFAULT uuid_generate_v7(),
jobId uuid not null,
foreign key (jobId) references jobs(id),
unique (jobId)
);
CREATE TABLE collectorVersions (
collectorId uuid not null,
jobId uuid not null,
id int not null,
addedAt timestamp not null default current_timestamp,
primary key (id, collectorId),
foreign key (collectorId) references collectors(id)
foreign key (jobId) references jobs(id),
primary key (id, jobId)
);
CREATE OR REPLACE FUNCTION setCollectorVersionNumber()
@@ -19,8 +12,8 @@ BEGIN
SELECT COALESCE(MAX(id), 0) + 1
INTO NEW.id
FROM collectorVersions
WHERE collectorId = NEW.collectorId;
WHERE jobId = NEW.jobId;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
@@ -32,20 +25,20 @@ EXECUTE FUNCTION setCollectorVersionNumber();
CREATE TABLE collectorActiveVersions (
id uuid primary key DEFAULT uuid_generate_v7(),
collectorId uuid not null,
jobId uuid not null,
versionId int not null,
foreign key (collectorId, versionId) references collectorVersions(collectorId, id)
foreign key (jobId, versionId) references collectorVersions(jobId, id)
);
CREATE TABLE collectorMinCleanVersions (
id uuid primary key DEFAULT uuid_generate_v7(),
collectorId uuid not null,
jobId uuid not null,
versionId int not null,
addedVersion int not null,
removedVersion int,
foreign key (collectorId, addedVersion) references collectorVersions(collectorId, id),
foreign key (collectorId, removedVersion) references collectorVersions(collectorId, id),
foreign key (collectorId) references collectors(id)
foreign key (jobId, addedVersion) references collectorVersions(jobId, id),
foreign key (jobId, removedVersion) references collectorVersions(jobId, id),
foreign key (jobId) references jobs(id)
);
CREATE OR REPLACE FUNCTION removeMinCleanVersion()
@@ -53,8 +46,8 @@ RETURNS TRIGGER AS $$
BEGIN
UPDATE collectorMinCleanVersions
SET removedVersion = NEW.addedVersion
WHERE collectorId = NEW.collectorId and removedVersion is null;
WHERE jobId = NEW.jobId and removedVersion is null;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
@@ -66,13 +59,13 @@ EXECUTE FUNCTION removeMinCleanVersion();
CREATE TABLE collectorMinTextVersions (
id uuid primary key DEFAULT uuid_generate_v7(),
collectorId uuid not null,
jobId uuid not null,
versionId int not null,
addedVersion int not null,
removedVersion int,
foreign key (collectorId, addedVersion) references collectorVersions(collectorId, id),
foreign key (collectorId, removedVersion) references collectorVersions(collectorId, id),
foreign key (collectorId) references collectors(id)
foreign key (jobId, addedVersion) references collectorVersions(jobId, id),
foreign key (jobId, removedVersion) references collectorVersions(jobId, id),
foreign key (jobId) references jobs(id)
);
CREATE OR REPLACE FUNCTION removeMinTextVersion()
@@ -80,8 +73,8 @@ RETURNS TRIGGER AS $$
BEGIN
UPDATE collectorMinTextVersions
SET removedVersion = NEW.addedVersion
WHERE collectorId = NEW.collectorId and removedVersion is null;
WHERE jobId = NEW.jobId and removedVersion is null;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
@@ -93,14 +86,14 @@ EXECUTE FUNCTION removeMinTextVersion();
CREATE TABLE collectorQueries (
id uuid primary key DEFAULT uuid_generate_v7(),
collectorId uuid not null,
jobId uuid not null,
name varchar(255) not null,
queryId uuid not null,
addedVersion int not null,
removedVersion int,
foreign key (queryId) references queries(id),
foreign key (collectorId) references collectors(id),
foreign key (collectorId, addedVersion) references collectorVersions(collectorId, id),
foreign key (collectorId, removedVersion) references collectorVersions(collectorId, id),
unique (collectorId, name, removedVersion)
);
foreign key (jobId) references jobs(id),
foreign key (jobId, addedVersion) references collectorVersions(jobId, id),
foreign key (jobId, removedVersion) references collectorVersions(jobId, id),
unique (jobId, name, removedVersion)
);
database/migrations/00000000000006_documents.up.sql
+2 -2
View File
@@ -25,9 +25,9 @@ CREATE TABLE documentCleans (
CREATE TABLE documentTextExtractions (
id uuid primary key DEFAULT uuid_generate_v7(),
documentId uuid not null,
cleanEntryId uuid not null,
version int not null,
bucket text not null,
key text not null,
foreign key (documentId) references documents(id)
foreign key (cleanEntryId) references documentCleans(id)
);
database/migrations/00000000000007_results.up.sql
+11 -6
View File
@@ -1,13 +1,18 @@
CREATE TABLE results (
id uuid primary key DEFAULT uuid_generate_v7(),
textEntryId uuid not null,
queryId uuid not null,
documentId uuid not null,
value TEXT not null,
cleanVersion int not null,
textVersion int not null,
queryVersion int not null,
foreign key (queryId) references queries(id),
foreign key (documentId) references documents(id),
unique (queryId, documentId, cleanVersion, textVersion, queryVersion),
foreign key (textEntryId) references documentTextExtractions(id),
foreign key (queryId, queryVersion) references queryVersions(queryId, id)
);
);
CREATE TABLE resultDependencies (
resultId uuid not null,
requiredResultId uuid not null,
foreign key (resultId) references results(id),
foreign key (requiredResultId) references results(id),
CONSTRAINT result_not_self_dependent CHECK (resultId != requiredResultId)
);
database/migrations/00000000000100_query_views.up.sql
+56 -44
View File
@@ -1,59 +1,71 @@
CREATE VIEW queryCurrentActiveVersions as
SELECT DISTINCT
av.queryId,
(FIRST_VALUE(av.versionId) OVER (PARTITION BY av.queryId ORDER BY av.id DESC))::int as id
FROM queryActiveVersions as av;
q.id as queryId,
coalesce(
(FIRST_VALUE(av.versionId) OVER (PARTITION BY q.id ORDER BY av.id DESC)),
0
)::int as activeVersion
FROM queries AS q
LEFT JOIN queryActiveVersions as av on av.queryId = q.id;
CREATE VIEW queryLatestVersions as
SELECT queryId, max(id)::int as id
FROM queryVersions
GROUP BY queryId;
SELECT
q.id as queryId,
coalesce(max(v.id), 0)::int as latestVersion
FROM queries AS q
LEFT JOIN queryVersions as v on v.queryId = q.id
GROUP BY q.id;
CREATE VIEW fullActiveQueries AS
with config as (
SELECT c.queryId, c.config
FROM queries AS q
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
LEFT JOIN queryConfigs AS c ON q.id = c.queryId
and isInVersion(av.id, c.addedVersion, c.removedVersion)
),
requiredIds as (
SELECT r.queryId,
ARRAY_AGG(DISTINCT r.requiredQueryId)
FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[]
as requiredIds
FROM queries AS q
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
and isInVersion(av.id, r.addedVersion, r.removedVersion)
GROUP BY r.queryId
)
SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config,
CREATE VIEW queryCurrentConfigs as
SELECT av.queryId, c.config
FROM queryCurrentActiveVersions as av
LEFT JOIN queryConfigs AS c ON av.queryId = c.queryId
and isInVersion(av.activeVersion, c.addedVersion, c.removedVersion);
CREATE VIEW queryCurrentRequiredIds as
SELECT DISTINCT av.queryId, r.requiredQueryId
FROM queryCurrentActiveVersions as av
LEFT JOIN requiredQueries AS r ON av.queryId = r.queryId
and isInVersion(av.activeVersion, r.addedVersion, r.removedVersion);
CREATE VIEW queryCurrentRequiredIdsAGG as
SELECT queryId,
coalesce(
r.requiredIds,
ARRAY_AGG(DISTINCT requiredQueryId)
FILTER (WHERE requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
array[]::uuid[]
)::uuid[] as requiredIds
FROM queryCurrentRequiredIds
GROUP BY queryId;
CREATE VIEW fullActiveQueries AS
SELECT DISTINCT q.id, q.type, av.activeVersion, lv.latestVersion, c.config, r.requiredIds
FROM queries AS q
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id
LEFT JOIN config AS c ON q.id = c.queryId
LEFT JOIN requiredIds AS r ON q.id = r.queryId;
JOIN queryCurrentActiveVersions as av on q.id = av.queryId
JOIN queryLatestVersions as lv on lv.queryId = q.id
JOIN queryCurrentConfigs AS c ON q.id = c.queryId
JOIN queryCurrentRequiredIdsAGG AS r ON q.id = r.queryId;
CREATE VIEW queryActiveDependencies AS
with recursive queryActiveDependencies AS (
SELECT q.id, r.requiredQueryId
FROM queries AS q
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
and isInVersion(av.id, r.addedVersion, r.removedVersion)
WITH RECURSIVE queryActiveDependencies(queryId, requiredQueryId, path, cycle) AS (
SELECT
queryId,
requiredQueryId,
ARRAY[queryId, requiredQueryId]::uuid[] AS path,
false AS cycle
FROM queryCurrentRequiredIds
UNION ALL
SELECT qd.id, r.requiredQueryId
FROM queries AS q
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
and isInVersion(av.id, r.addedVersion, r.removedVersion)
JOIN queryActiveDependencies as qd on q.id = qd.requiredQueryId
SELECT
q.queryId,
qd.requiredQueryId,
path || qd.requiredQueryId,
qd.requiredQueryId = ANY(path) AS cycle
FROM queryCurrentRequiredIds as q
JOIN queryActiveDependencies as qd ON q.queryId = qd.requiredQueryId
WHERE NOT qd.cycle -- Stop if we detect a cycle
)
SELECT DISTINCT id, requiredQueryId FROM queryActiveDependencies;
SELECT DISTINCT queryId as id, requiredQueryId
FROM queryActiveDependencies
WHERE NOT cycle;
database/migrations/00000000000101_collector_views.up.sql
+65 -35
View File
@@ -1,47 +1,77 @@
CREATE VIEW collectorCurrentActiveVersions as
SELECT DISTINCT
av.collectorId,
(FIRST_VALUE(av.versionId) OVER (PARTITION BY av.collectorId ORDER BY av.id DESC))::int as id
FROM collectorActiveVersions as av;
j.id as jobId,
coalesce(
(FIRST_VALUE(v.versionId) OVER (PARTITION BY j.id ORDER BY v.id DESC)),
0
)::int as activeVersion
FROM jobs as j
LEFT JOIN collectorActiveVersions as v on v.jobId = j.id;
CREATE VIEW collectorLatestVersions as
SELECT collectorId, max(id)::int as id
FROM collectorVersions
GROUP BY collectorId;
SELECT
j.id as jobId,
coalesce(max(v.id), 0)::int as latestVersion
FROM jobs as j
LEFT JOIN collectorVersions as v on v.jobId = j.id
GROUP BY j.id;
CREATE VIEW currentCollectorMinTextVersions as
SELECT DISTINCT
av.jobId,
coalesce(ctv.versionId, 0) as minTextVersion
FROM collectorCurrentActiveVersions as av
LEFT JOIN collectorMinTextVersions AS ctv ON av.jobId = ctv.jobId
and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
CREATE VIEW currentCollectorMinCleanVersions as
SELECT DISTINCT
av.jobId,
coalesce(ctv.versionId, 0) as minCleanVersion
FROM collectorCurrentActiveVersions as av
LEFT JOIN collectorMinCleanVersions AS ctv ON av.jobId = ctv.jobId
and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
CREATE VIEW currentCollectorQueries as
SELECT DISTINCT
av.jobId,
q.name,
q.queryId
FROM collectorCurrentActiveVersions as av
LEFT JOIN collectorQueries AS q ON av.jobId = q.jobId
and isInVersion(av.activeVersion, q.addedVersion, q.removedVersion);
CREATE VIEW currentCollectorQueriesJSONAGG as
SELECT DISTINCT
jobId,
jsonb_object_agg(name, queryId) FILTER (WHERE name is not null) AS fields
FROM currentCollectorQueries
GROUP BY jobId;
CREATE VIEW fullActiveCollectors AS
SELECT DISTINCT c.id, c.jobId, coalesce(ccv.versionId, 0) as minCleanVersion, coalesce(ctv.versionId, 0) as minTextVersion, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion,
jsonb_object_agg(q.name, q.queryId) FILTER (WHERE q.name is not null) AS fields
FROM collectors AS c
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorLatestVersions as lv on lv.collectorId = c.id
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId
and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion)
LEFT JOIN collectorQueries AS q ON c.id = q.collectorId
and isInVersion(av.id, q.addedVersion, q.removedVersion)
GROUP BY c.id, c.jobId, ccv.versionId, ctv.versionId, av.id, lv.id;
SELECT DISTINCT av.jobId,
ccv.minCleanVersion, ctv.minTextVersion,
av.activeVersion, lv.latestVersion,
q.fields
FROM collectorCurrentActiveVersions as av
JOIN collectorLatestVersions as lv on lv.jobId = av.jobId
JOIN currentCollectorMinCleanVersions AS ccv ON av.jobId = ccv.jobId
JOIN currentCollectorMinTextVersions AS ctv ON av.jobId = ctv.jobId
JOIN currentCollectorQueriesJSONAGG AS q ON av.jobId = q.jobId;
CREATE VIEW collectorQueryDependencyTree AS
WITH RECURSIVE
coll as (
SELECT DISTINCT c.id, c.jobId, q.queryId
FROM collectors as c
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorQueries as q ON c.id = q.collectorId
and isInVersion(av.id, q.addedVersion, q.removedVersion)
),
collectorQueryDependencyTree AS (
SELECT cq.id as collectorId, cq.jobId, aqc.id as queryId, aqc.type, aqc.requiredIds, aqc.activeVersion
FROM coll as cq
JOIN fullActiveQueries as aqc on cq.queryId = aqc.id
WITH RECURSIVE collectorQueryDependencyTree AS (
SELECT cq.jobId, cq.queryId, ri.requiredIds
FROM currentCollectorQueries as cq
JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId
UNION ALL
SELECT acq.collectorId, acq.jobId, q.id as queryId, q.type, q.requiredIds, q.activeVersion
FROM fullActiveQueries as q
JOIN collectorQueryDependencyTree as acq on q.id = ANY(acq.requiredIds)
SELECT acq.jobId, q.queryId, q.requiredIds
FROM queryCurrentRequiredIdsAGG as q
JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds)
)
SELECT DISTINCT collectorId, jobId, queryId, type, activeVersion as queryVersion, requiredIds
FROM collectorQueryDependencyTree;
SELECT DISTINCT ct.jobId, ct.queryId, q.type, av.activeVersion as queryVersion, ct.requiredIds
FROM collectorQueryDependencyTree as ct
JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId
JOIN queries as q on q.id = ct.queryId;
database/migrations/00000000000102_job_views.up.sql
+81 -2
View File
@@ -8,7 +8,7 @@ DECLARE
totalCount BIGINT := 0;
BEGIN
SELECT COUNT(*)::BIGINT INTO totalCount FROM documents WHERE jobId = _jobId;
RETURN QUERY
SELECT d.id, totalCount
FROM documents as d
@@ -17,4 +17,83 @@ BEGIN
LIMIT _batchSize
OFFSET _offset;
END;
$$ LANGUAGE plpgsql;
$$ LANGUAGE plpgsql;
CREATE VIEW currentCleanEntries as
WITH RankedExtractions AS (
SELECT
dc.id,
dc.documentId,
dc.bucket,
dc.key,
dc.version,
ROW_NUMBER() OVER (PARTITION BY dc.documentId ORDER BY dc.id DESC) as row_num
FROM documents d
JOIN currentCollectorMinCleanVersions ccv ON d.jobId = ccv.jobId
JOIN documentCleans dc ON dc.documentId = d.id
AND dc.version >= ccv.minCleanVersion
)
SELECT
id,
documentId,
bucket,
key,
version
FROM RankedExtractions
WHERE row_num = 1;
CREATE VIEW currentTextEntries as
WITH RankedExtractions AS (
SELECT
dte.id,
cc.documentId,
dte.bucket,
dte.key,
dte.version,
dte.cleanEntryId,
ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dte.id DESC) as row_num
FROM documents d
JOIN currentCleanEntries cc on cc.documentId = d.id
JOIN currentCollectorMinTextVersions ctv ON ctv.jobId = d.jobId
JOIN documentTextExtractions dte ON dte.cleanEntryId = cc.id
AND dte.version >= ctv.minTextVersion
)
SELECT
id,
documentId,
bucket,
key,
version,
cleanEntryId
FROM RankedExtractions
WHERE row_num = 1;
CREATE VIEW currentClientCanSync as
WITH RankedExtractions AS (
SELECT
ccs.clientId,
ccs.canSync,
ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.id DESC) as row_num
FROM clients c
JOIN clientCanSync ccs on c.id = ccs.clientId
)
SELECT
c.id as clientId,
coalesce(r.canSync, false) as canSync
FROM clients c
LEFT JOIN RankedExtractions r on r.clientId = c.id and r.row_num = 1;
CREATE VIEW currentJobCanSync as
WITH RankedExtractions AS (
SELECT
ccs.jobId,
ccs.canSync,
ROW_NUMBER() OVER (PARTITION BY ccs.jobId ORDER BY ccs.id DESC) as row_num
FROM jobs c
JOIN jobCanSync ccs on c.id = ccs.jobId
)
SELECT
c.id as jobId,
coalesce(r.canSync, false) as canSync
FROM jobs c
LEFT JOIN RankedExtractions r on r.jobId = c.id and r.row_num = 1;
database/queries/clean.sql
+4 -17
View File
@@ -1,25 +1,12 @@
-- name: IsDocumentClean :one
SELECT EXISTS(
SELECT 1
FROM documentCleans AS dc
JOIN documents AS d ON d.id = dc.documentId
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
SELECT 1 FROM currentCleanEntries WHERE documentId = $1
);
-- name: AddDocumentCleanEntry :exec
INSERT INTO documentCleans (documentId, version, bucket, key) VALUES ($1, $2, $3, $4);
-- name: GetDocumentCleanEntry :one
SELECT dc.documentId, dc.bucket, dc.key, dc.version
FROM documentCleans AS dc
JOIN documents AS d ON d.id = dc.documentId
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
ORDER BY d.id DESC LIMIT 1;
SELECT id, documentId, bucket, key, version
FROM currentCleanEntries
WHERE documentId = $1;
database/queries/client.sql
+2 -8
View File
@@ -2,15 +2,9 @@
INSERT INTO clients (name) VALUES ($1) RETURNING id;
-- name: GetClient :one
SELECT c.id, c.name, coalesce(cs.canSync, false) as canSync
SELECT c.id, c.name, cs.canSync
FROM clients as c
LEFT JOIN (
SELECT clientId, canSync
FROM clientCanSync
WHERE clientId = $1
ORDER BY id DESC
LIMIT 1
) as cs on cs.clientId = c.id
JOIN currentClientCanSync as cs on cs.clientId = c.id
WHERE c.id = $1;
-- name: UpdateClient :exec
database/queries/collector.sql
+9 -12
View File
@@ -1,29 +1,26 @@
-- name: ListCollectorQueries :many
SELECT * FROM collectorQueryDependencyTree WHERE collectorId = $1;
SELECT * FROM collectorQueryDependencyTree WHERE jobId = @jobId;
-- name: GetCollectorByJobID :one
SELECT * FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1;
SELECT * FROM fullActiveCollectors WHERE jobId = @jobId LIMIT 1;
-- name: GetCollector :one
SELECT * FROM fullActiveCollectors WHERE id = $1 LIMIT 1;
-- name: CreateCollector :one
INSERT INTO collectors (jobId) VALUES ($1) RETURNING id;
SELECT * FROM fullActiveCollectors WHERE jobId = @jobId LIMIT 1;
-- name: AddLatestCollectorVersion :one
INSERT INTO collectorVersions (collectorId) VALUES ($1) RETURNING id;
INSERT INTO collectorVersions (jobId) VALUES ($1) RETURNING id;
-- name: SetActiveCollectorVersion :exec
INSERT INTO collectorActiveVersions (collectorId, versionId) VALUES ($1, $2);
INSERT INTO collectorActiveVersions (jobId, versionId) VALUES ($1, $2);
-- name: SetCollectorCleanVersion :exec
INSERT INTO collectorMinCleanVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3);
INSERT INTO collectorMinCleanVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3);
-- name: SetCollectorTextVersion :exec
INSERT INTO collectorMinTextVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3);
INSERT INTO collectorMinTextVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3);
-- name: AddCollectorQuery :exec
INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4);
INSERT INTO collectorQueries (jobId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4);
-- name: RemoveCollectorQuery :exec
UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and collectorId = $3 and removedVersion is null;
UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and jobId = $3 and removedVersion is null;
database/queries/job.sql
+78 -8
View File
@@ -1,13 +1,7 @@
-- name: GetJob :one
SELECT j.id, j.clientId, coalesce(cs.canSync, false) as canSync
SELECT j.id, j.clientId, cs.canSync
FROM jobs as j
LEFT JOIN (
SELECT jobId, canSync
FROM jobCanSync
WHERE jobId = $1
ORDER BY id DESC
LIMIT 1
) as cs on cs.jobId = j.id
JOIN currentJobCanSync as cs on cs.jobId = j.id
WHERE j.id = $1;
-- name: CreateJob :one
@@ -18,3 +12,79 @@ INSERT INTO jobCanSync (canSync, jobId) VALUES ($1, $2);
-- name: ListJobDocumentIDsBatch :many
SELECT id, totalCount FROM listJobDocumentIDs(@jobId, @batchSize, @pageOffset);
-- name: IsJobSynced :one
WITH
docs AS (
-- Get all documents for this job
SELECT id, jobId FROM documents WHERE jobId = $1
),
doc_text_entries AS (
-- Documents with their current text entries
SELECT
d.id AS document_id,
cte.id AS text_entry_id
FROM
docs d
LEFT JOIN currentTextEntries cte ON cte.documentId = d.id
),
required_results AS (
-- All required document-query-version combinations
SELECT
d.id AS document_id,
cqdt.queryId,
cqdt.queryVersion,
dte.text_entry_id
FROM
docs d
JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.jobId
JOIN doc_text_entries dte ON dte.document_id = d.id
and dte.text_entry_id IS NOT NULL
),
existing_results AS (
-- Valid results that exist
SELECT
rr.document_id AS document_id,
r.queryId,
r.id AS result_id
FROM
results r
JOIN required_results rr ON
r.queryId = rr.queryId AND
r.queryVersion = rr.queryVersion AND
r.textEntryId = rr.text_entry_id
),
missing_results AS (
-- Find missing results
SELECT rr.queryId
FROM required_results rr
LEFT JOIN existing_results er on er.queryId = rr.queryId
WHERE er.result_id is null
),
dependency_check AS (
-- Check for missing dependencies
SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId
FROM existing_results er
JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId
LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id
WHERE rd.resultId is null
)
SELECT (
-- No documents means job is synced
NOT EXISTS (SELECT 1 FROM docs)
OR
-- Documents with no text entries
(NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL)
and
-- Documents with missing results
NOT EXISTS (SELECT 1 FROM missing_results)
and
-- Documents with missing dependencies
NOT EXISTS (SELECT 1 FROM dependency_check))
)::bool as is_synced;
database/queries/query.sql
+22 -17
View File
@@ -1,5 +1,5 @@
-- name: GetQueryConfig :one
SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 and COALESCE(removedVersion, $2 - 1) < $2;
-- name: GetActiveQueryConfig :one
SELECT config FROM queryCurrentConfigs where queryId = $1;
-- name: GetQuery :one
SELECT * FROM fullActiveQueries WHERE id = $1;
@@ -16,22 +16,23 @@ SELECT c.queryId, c.config
),
requiredIds as (
SELECT r.queryId,
coalesce(
ARRAY_AGG(DISTINCT r.requiredQueryId)
FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[]
as requiredIds
FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
array[]::uuid[]
)::uuid[] as requiredIds
FROM query AS q
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
and isInVersion(@version, r.addedVersion, r.removedVersion)
GROUP BY r.queryId
)
SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config,
coalesce(
r.requiredIds,
array[]::uuid[]
)::uuid[] as requiredIds
SELECT DISTINCT q.id, q.type,
av.activeVersion,
lv.latestVersion, c.config,
r.requiredIds
FROM query AS q
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id
JOIN queryCurrentActiveVersions as av on q.id = av.queryId
JOIN queryLatestVersions as lv on lv.queryId = q.id
LEFT JOIN config AS c ON q.id = c.queryId
LEFT JOIN requiredIds AS r ON q.id = r.queryId;
@@ -66,17 +67,21 @@ SELECT COUNT(*) = COUNT(DISTINCT id) AS all_exist
-- name: IsQueryInDependencyTree :one
SELECT EXISTS (
SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1)
SELECT 1 FROM queryActiveDependencies
WHERE id = any(@requiredQueryIds)
and requiredQueryId = @queryId
or @queryId = any(@requiredQueryIds)
);
-- name: ListQueryDirectDependentsByDocumentID :many
WITH doc AS (
SELECT id, jobId FROM documents where id = $2
SELECT id, jobId FROM documents where id = @documentId
)
SELECT dt.queryId
FROM collectorQueryDependencyTree as dt
JOIN doc as d on d.jobId = dt.jobId
where $1 = any(dt.requiredIds);
FROM doc as d
JOIN collectorQueryDependencyTree as dt
on d.jobId = dt.jobId
and @queryId = any(dt.requiredIds);
-- name: ListQueryJobIDs :many
SELECT jobId FROM collectorQueryDependencyTree WHERE queryId = $1;
SELECT jobId FROM collectorQueryDependencyTree WHERE queryId = $1;
database/queries/result.sql
+41 -39
View File
@@ -1,9 +1,9 @@
-- name: ListQueryRequirementValues :many
WITH reqQueries as (
SELECT q.id as queryId, av.id as activeVersion, q.type
SELECT av.queryId, av.activeVersion, q.type
FROM requiredQueries as rq
JOIN queries as q on q.id = rq.requiredQueryId
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId
JOIN queries as q on q.id = av.queryId
WHERE rq.queryId = @queryId
and isInVersion(@version, rq.addedVersion, rq.removedVersion)
),
@@ -13,70 +13,72 @@ docs as (
WHERE id = @documentId
),
codeVersions as (
SELECT
SELECT
d.id as documentId,
coalesce(ccv.versionId, 1) as minCleanVersion,
coalesce(ctv.versionId, 1) as minTextVersion
mcv.minCleanVersion,
mtv.minTextVersion
FROM docs as d
LEFT JOIN collectors as c on c.jobId = d.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId
and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion)
LIMIT 1
JOIN currentCollectorMinTextVersions as mtv on mtv.jobId = d.jobId
JOIN currentCollectorMinCleanVersions as mcv on mcv.jobId = d.jobId
),
latestVersions AS (
SELECT
SELECT
r.id,
rq.queryId,
rq.type,
r.queryVersion,
r.cleanVersion,
r.textVersion,
r.textEntryId,
r.value,
ROW_NUMBER() OVER (
PARTITION BY rq.queryId
ORDER BY r.cleanVersion DESC, r.textVersion DESC
PARTITION BY r.queryId
ORDER BY r.id DESC
) as rowNumber
FROM reqQueries as rq
JOIN currentTextEntries as cte on cte.documentId = @documentId
LEFT JOIN results as r ON rq.queryId = r.queryId
and r.documentId = @documentId
and r.queryVersion = rq.activeVersion
JOIN codeVersions as ccv on ccv.documentId = r.documentId
and r.cleanVersion >= ccv.minCleanVersion
and r.textVersion >= ccv.minTextVersion
and cte.id = r.textEntryId
)
SELECT DISTINCT lv.queryId, lv.type, r.value
FROM latestVersions as lv
JOIN results as r ON r.queryId = lv.queryId
and r.documentId = @documentId
and r.queryVersion = lv.queryVersion
and r.cleanVersion = lv.cleanVersion
and r.textVersion = lv.textVersion
and lv.rowNumber = 1;
SELECT DISTINCT id, queryId, type, value
FROM latestVersions
WHERE rowNumber = 1;
-- name: SetResult :exec
INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6);
-- name: AddResult :one
INSERT INTO results (queryId, value, textEntryId, queryVersion) VALUES ($1, $2, $3, $4) returning id;
-- name: AddResultDependency :exec
INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2);
-- name: GetResultValueWithVersion :one
SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5;
WITH doc as (
SELECT id, jobId
FROM documents
WHERE id = @documentId
)
SELECT r.id, r.value
FROM doc as d
JOIN currentTextEntries as cte on cte.documentId = d.id
LEFT JOIN results as r
on r.queryId = @queryId
and r.queryVersion = @queryVersion
and r.textEntryId = cte.id;
-- name: ListUnsyncedNoDepsQueriesByDocId :many
WITH docs as (
SELECT id, jobId from documents where id = $1
),
unsyncedQueries AS (
SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id
SELECT DISTINCT dt.queryId, dt.requiredIds
from docs as d
JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId
JOIN fullActiveCollectors as c on c.id = dt.collectorId
LEFT JOIN results as r on r.queryId = dt.queryId
and r.documentId = d.id
JOIN currentTextEntries as cte on cte.documentId = d.id
LEFT JOIN results as r
on r.queryId = dt.queryId
and r.queryVersion = dt.queryVersion
and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
and cte.id = r.textEntryId
where r.value is null
)
SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
WHERE NOT EXISTS (
SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds)
);
database/queries/text.sql
+5 -18
View File
@@ -1,25 +1,12 @@
-- name: IsDocumentTextExtracted :one
SELECT EXISTS(
SELECT 1
FROM documentTextExtractions AS dc
JOIN documents AS d ON d.id = dc.documentId
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
SELECT 1 FROM currentTextEntries WHERE documentId = @documentId
);
-- name: AddDocumentTextEntry :exec
INSERT INTO documentTextExtractions (documentId, version, bucket, key) VALUES ($1, $2, $3, $4);
INSERT INTO documentTextExtractions (version, bucket, key, cleanEntryId) VALUES ($1, $2, $3, $4);
-- name: GetDocumentTextEntry :one
SELECT dc.documentId, dc.bucket, dc.key, dc.version
FROM documentTextExtractions AS dc
JOIN documents AS d ON d.id = dc.documentId
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
ORDER BY d.id DESC LIMIT 1;
SELECT id, documentId, bucket, key, version, cleanEntryId
FROM currentTextEntries
WHERE documentId = @documentId;
devbox.json
+37 -36
View File
@@ -1,5 +1,38 @@
{
"$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.13.7/.schema/devbox.schema.json",
"env": {
"APP_ENV": "development",
"AWS_ACCESS_KEY_ID": "test",
"AWS_ENDPOINT_URL": "http://localhost:4566",
"AWS_REGION": "us-east-1",
"AWS_SECRET_ACCESS_KEY": "test",
"AWS_SESSION_TOKEN": "",
"BUCKET_IN": "documentin",
"DB_NOSSL": "true",
"DOCUMENT_CLEAN_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_clean",
"DOCUMENT_INIT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_init",
"DOCUMENT_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_sync",
"DOCUMENT_TEXT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_text",
"JOB_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/job_sync",
"PGDATABASE": "query_orchestration",
"PGHOST": "localhost",
"PGPASSWORD": "pass",
"PGPORT": "5432",
"PGPORT_GENERATE": "5431",
"PGUSER": "postgres",
"QNAME_DOCUMENT_CLEAN": "document_clean",
"QNAME_DOCUMENT_INIT": "document_init",
"QNAME_DOCUMENT_SYNC": "document_sync",
"QNAME_DOCUMENT_TEXT": "document_text",
"QNAME_JOB_SYNC": "job_sync",
"QNAME_QUERY_RUNNER": "query_runner",
"QNAME_QUERY_SYNC": "query_sync",
"QNAME_QUERY_VERSION_SYNC": "query_version_sync",
"QUERY_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_sync",
"QUERY_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_runner",
"QUERY_VERSION_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_version_sync"
},
"env_from": ".env",
"packages": [
"golangci-lint@1.62.2",
"protolint@0.50.5",
@@ -18,7 +51,8 @@
"oapi-codegen@2.4.1",
"vacuum-go@0.14.1",
"awscli2@2.19.0",
"postgresql@17.2"
"postgresql@17.2",
"bc@1.07.1"
],
"shell": {
"init_hook": [
@@ -26,38 +60,5 @@
"export DB_URI_GENERATE=postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT_GENERATE}/${PGDATABASE}?sslmode=disable",
"echo 'Welcome to the DoczyAI devbox!'"
]
},
"env": {
"APP_ENV": "development",
"PGUSER": "postgres",
"PGPASSWORD": "pass",
"PGHOST": "localhost",
"PGPORT": "5432",
"PGPORT_GENERATE": "5431",
"PGDATABASE": "query_orchestration",
"DB_NOSSL": "true",
"AWS_ACCESS_KEY_ID": "test",
"AWS_SECRET_ACCESS_KEY": "test",
"AWS_SESSION_TOKEN": "",
"AWS_REGION": "us-east-1",
"AWS_ENDPOINT_URL": "http://localhost:4566",
"QNAME_DOCUMENT_INIT": "document_init",
"QNAME_DOCUMENT_SYNC": "document_sync",
"QNAME_DOCUMENT_CLEAN": "document_clean",
"QNAME_DOCUMENT_TEXT": "document_text",
"QNAME_QUERY_SYNC": "query_sync",
"QNAME_QUERY_RUNNER": "query_runner",
"QNAME_JOB_SYNC": "job_sync",
"QNAME_QUERY_VERSION_SYNC": "query_version_sync",
"DOCUMENT_INIT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_init",
"DOCUMENT_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_sync",
"DOCUMENT_CLEAN_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_clean",
"DOCUMENT_TEXT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_text",
"QUERY_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_sync",
"QUERY_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_runner",
"JOB_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/job_sync",
"QUERY_VERSION_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_version_sync",
"BUCKET_IN": "documentin"
},
"env_from": ".env"
}
}
}
devbox.lock
+27 -10
View File
@@ -65,34 +65,51 @@
}
}
},
"github:NixOS/nixpkgs/nixpkgs-unstable": {
"resolved": "github:NixOS/nixpkgs/b1b43d32be000928cc71250ed77f4a0a5f2bc23a?lastModified=1739698114&narHash=sha256-8S9n69Dnpg8DhfFlP0YvMGmSOY2X4kImGSPWXYNpaHM%3D"
},
"glibcLocales@latest": {
"bc@1.07.1": {
"last_modified": "2025-02-07T11:26:36Z",
"resolved": "github:NixOS/nixpkgs/d98abf5cf5914e5e4e9d57205e3af55ca90ffc1d#glibcLocales",
"resolved": "github:NixOS/nixpkgs/d98abf5cf5914e5e4e9d57205e3af55ca90ffc1d#bc",
"source": "devbox-search",
"version": "2.40-66",
"version": "1.07.1",
"systems": {
"aarch64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/vbpa5cim632v5m8lkd3jx6gfmny1m6vf-bc-1.07.1",
"default": true
}
],
"store_path": "/nix/store/vbpa5cim632v5m8lkd3jx6gfmny1m6vf-bc-1.07.1"
},
"aarch64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/msl6h7i0mgzayw4icb58hccfkh9672bl-glibc-locales-2.40-66",
"path": "/nix/store/49rm0gkxa9mrfnvzvgnfmcmwim3ilc16-bc-1.07.1",
"default": true
}
],
"store_path": "/nix/store/msl6h7i0mgzayw4icb58hccfkh9672bl-glibc-locales-2.40-66"
"store_path": "/nix/store/49rm0gkxa9mrfnvzvgnfmcmwim3ilc16-bc-1.07.1"
},
"x86_64-darwin": {
"outputs": [
{
"name": "out",
"path": "/nix/store/4b3aqnqpnqd8j425f0isg4bfr56wiyfr-bc-1.07.1",
"default": true
}
],
"store_path": "/nix/store/4b3aqnqpnqd8j425f0isg4bfr56wiyfr-bc-1.07.1"
},
"x86_64-linux": {
"outputs": [
{
"name": "out",
"path": "/nix/store/p1fn0y9zbq7fng8cn49cvinqqh47slax-glibc-locales-2.40-66",
"path": "/nix/store/4bfbksfiy1b6r6yna9hh39qv8jfi33n7-bc-1.07.1",
"default": true
}
],
"store_path": "/nix/store/p1fn0y9zbq7fng8cn49cvinqqh47slax-glibc-locales-2.40-66"
"store_path": "/nix/store/4bfbksfiy1b6r6yna9hh39qv8jfi33n7-bc-1.07.1"
}
}
},
internal/database/repository/clean.sql.go
+11 -43
View File
@@ -36,39 +36,21 @@ func (q *Queries) AddDocumentCleanEntry(ctx context.Context, arg *AddDocumentCle
}
const getDocumentCleanEntry = `-- name: GetDocumentCleanEntry :one
SELECT dc.documentId, dc.bucket, dc.key, dc.version
FROM documentCleans AS dc
JOIN documents AS d ON d.id = dc.documentId
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
ORDER BY d.id DESC LIMIT 1
SELECT id, documentId, bucket, key, version
FROM currentCleanEntries
WHERE documentId = $1
`
type GetDocumentCleanEntryRow struct {
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
}
// GetDocumentCleanEntry
//
// SELECT dc.documentId, dc.bucket, dc.key, dc.version
// FROM documentCleans AS dc
// JOIN documents AS d ON d.id = dc.documentId
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
// LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
// WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
// ORDER BY d.id DESC LIMIT 1
func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.UUID) (*GetDocumentCleanEntryRow, error) {
// SELECT id, documentId, bucket, key, version
// FROM currentCleanEntries
// WHERE documentId = $1
func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.UUID) (*Currentcleanentry, error) {
row := q.db.QueryRow(ctx, getDocumentCleanEntry, documentid)
var i GetDocumentCleanEntryRow
var i Currentcleanentry
err := row.Scan(
&i.ID,
&i.Documentid,
&i.Bucket,
&i.Key,
@@ -79,28 +61,14 @@ func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.U
const isDocumentClean = `-- name: IsDocumentClean :one
SELECT EXISTS(
SELECT 1
FROM documentCleans AS dc
JOIN documents AS d ON d.id = dc.documentId
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
SELECT 1 FROM currentCleanEntries WHERE documentId = $1
)
`
// IsDocumentClean
//
// SELECT EXISTS(
// SELECT 1
// FROM documentCleans AS dc
// JOIN documents AS d ON d.id = dc.documentId
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
// LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
// WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
// SELECT 1 FROM currentCleanEntries WHERE documentId = $1
// )
func (q *Queries) IsDocumentClean(ctx context.Context, documentid pgtype.UUID) (bool, error) {
row := q.db.QueryRow(ctx, isDocumentClean, documentid)
internal/database/repository/clean_test.go
+7 -6
View File
@@ -9,6 +9,7 @@ import (
"queryorchestration/internal/test"
"testing"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
)
@@ -65,10 +66,10 @@ func TestClean(t *testing.T) {
clean, err := queries.GetDocumentCleanEntry(ctx, id)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.GetDocumentCleanEntryRow{
Documentid: id,
Bucket: bucket,
Key: key,
Version: 1,
}, clean)
assert.Equal(t, id, clean.Documentid)
assert.Equal(t, bucket, clean.Bucket)
assert.Equal(t, key, clean.Key)
assert.Equal(t, int32(1), clean.Version)
assert.NotEqual(t, pgtype.UUID{}, clean.ID)
assert.True(t, clean.ID.Valid)
}
internal/database/repository/client.sql.go
+6 -18
View File
@@ -43,15 +43,9 @@ func (q *Queries) CreateClient(ctx context.Context, name string) (pgtype.UUID, e
}
const getClient = `-- name: GetClient :one
SELECT c.id, c.name, coalesce(cs.canSync, false) as canSync
SELECT c.id, c.name, cs.canSync
FROM clients as c
LEFT JOIN (
SELECT clientId, canSync
FROM clientCanSync
WHERE clientId = $1
ORDER BY id DESC
LIMIT 1
) as cs on cs.clientId = c.id
JOIN currentClientCanSync as cs on cs.clientId = c.id
WHERE c.id = $1
`
@@ -63,18 +57,12 @@ type GetClientRow struct {
// GetClient
//
// SELECT c.id, c.name, coalesce(cs.canSync, false) as canSync
// SELECT c.id, c.name, cs.canSync
// FROM clients as c
// LEFT JOIN (
// SELECT clientId, canSync
// FROM clientCanSync
// WHERE clientId = $1
// ORDER BY id DESC
// LIMIT 1
// ) as cs on cs.clientId = c.id
// JOIN currentClientCanSync as cs on cs.clientId = c.id
// WHERE c.id = $1
func (q *Queries) GetClient(ctx context.Context, clientid pgtype.UUID) (*GetClientRow, error) {
row := q.db.QueryRow(ctx, getClient, clientid)
func (q *Queries) GetClient(ctx context.Context, id pgtype.UUID) (*GetClientRow, error) {
row := q.db.QueryRow(ctx, getClient, id)
var i GetClientRow
err := row.Scan(&i.ID, &i.Name, &i.Cansync)
return &i, err
internal/database/repository/collector.sql.go
+35 -52
View File
@@ -12,11 +12,11 @@ import (
)
const addCollectorQuery = `-- name: AddCollectorQuery :exec
INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
INSERT INTO collectorQueries (jobId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
`
type AddCollectorQueryParams struct {
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Name string `db:"name"`
Queryid pgtype.UUID `db:"queryid"`
Addedversion int32 `db:"addedversion"`
@@ -24,10 +24,10 @@ type AddCollectorQueryParams struct {
// AddCollectorQuery
//
// INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
// INSERT INTO collectorQueries (jobId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
func (q *Queries) AddCollectorQuery(ctx context.Context, arg *AddCollectorQueryParams) error {
_, err := q.db.Exec(ctx, addCollectorQuery,
arg.Collectorid,
arg.Jobid,
arg.Name,
arg.Queryid,
arg.Addedversion,
@@ -36,45 +36,30 @@ func (q *Queries) AddCollectorQuery(ctx context.Context, arg *AddCollectorQueryP
}
const addLatestCollectorVersion = `-- name: AddLatestCollectorVersion :one
INSERT INTO collectorVersions (collectorId) VALUES ($1) RETURNING id
INSERT INTO collectorVersions (jobId) VALUES ($1) RETURNING id
`
// AddLatestCollectorVersion
//
// INSERT INTO collectorVersions (collectorId) VALUES ($1) RETURNING id
func (q *Queries) AddLatestCollectorVersion(ctx context.Context, collectorid pgtype.UUID) (int32, error) {
row := q.db.QueryRow(ctx, addLatestCollectorVersion, collectorid)
// INSERT INTO collectorVersions (jobId) VALUES ($1) RETURNING id
func (q *Queries) AddLatestCollectorVersion(ctx context.Context, jobid pgtype.UUID) (int32, error) {
row := q.db.QueryRow(ctx, addLatestCollectorVersion, jobid)
var id int32
err := row.Scan(&id)
return id, err
}
const createCollector = `-- name: CreateCollector :one
INSERT INTO collectors (jobId) VALUES ($1) RETURNING id
`
// CreateCollector
//
// INSERT INTO collectors (jobId) VALUES ($1) RETURNING id
func (q *Queries) CreateCollector(ctx context.Context, jobid pgtype.UUID) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, createCollector, jobid)
var id pgtype.UUID
err := row.Scan(&id)
return id, err
}
const getCollector = `-- name: GetCollector :one
SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE id = $1 LIMIT 1
SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
`
// GetCollector
//
// SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE id = $1 LIMIT 1
func (q *Queries) GetCollector(ctx context.Context, id pgtype.UUID) (*Fullactivecollector, error) {
row := q.db.QueryRow(ctx, getCollector, id)
// SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
func (q *Queries) GetCollector(ctx context.Context, jobid pgtype.UUID) (*Fullactivecollector, error) {
row := q.db.QueryRow(ctx, getCollector, jobid)
var i Fullactivecollector
err := row.Scan(
&i.ID,
&i.Jobid,
&i.Mincleanversion,
&i.Mintextversion,
@@ -86,17 +71,16 @@ func (q *Queries) GetCollector(ctx context.Context, id pgtype.UUID) (*Fullactive
}
const getCollectorByJobID = `-- name: GetCollectorByJobID :one
SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
`
// GetCollectorByJobID
//
// SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
// SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
func (q *Queries) GetCollectorByJobID(ctx context.Context, jobid pgtype.UUID) (*Fullactivecollector, error) {
row := q.db.QueryRow(ctx, getCollectorByJobID, jobid)
var i Fullactivecollector
err := row.Scan(
&i.ID,
&i.Jobid,
&i.Mincleanversion,
&i.Mintextversion,
@@ -108,14 +92,14 @@ func (q *Queries) GetCollectorByJobID(ctx context.Context, jobid pgtype.UUID) (*
}
const listCollectorQueries = `-- name: ListCollectorQueries :many
SELECT collectorid, jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
SELECT jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE jobId = $1
`
// ListCollectorQueries
//
// SELECT collectorid, jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.UUID) ([]*Collectorquerydependencytree, error) {
rows, err := q.db.Query(ctx, listCollectorQueries, collectorid)
// SELECT jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE jobId = $1
func (q *Queries) ListCollectorQueries(ctx context.Context, jobid pgtype.UUID) ([]*Collectorquerydependencytree, error) {
rows, err := q.db.Query(ctx, listCollectorQueries, jobid)
if err != nil {
return nil, err
}
@@ -124,7 +108,6 @@ func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.U
for rows.Next() {
var i Collectorquerydependencytree
if err := rows.Scan(
&i.Collectorid,
&i.Jobid,
&i.Queryid,
&i.Type,
@@ -142,72 +125,72 @@ func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.U
}
const removeCollectorQuery = `-- name: RemoveCollectorQuery :exec
UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and collectorId = $3 and removedVersion is null
UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and jobId = $3 and removedVersion is null
`
type RemoveCollectorQueryParams struct {
Removedversion *int32 `db:"removedversion"`
Queryid pgtype.UUID `db:"queryid"`
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
}
// RemoveCollectorQuery
//
// UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and collectorId = $3 and removedVersion is null
// UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and jobId = $3 and removedVersion is null
func (q *Queries) RemoveCollectorQuery(ctx context.Context, arg *RemoveCollectorQueryParams) error {
_, err := q.db.Exec(ctx, removeCollectorQuery, arg.Removedversion, arg.Queryid, arg.Collectorid)
_, err := q.db.Exec(ctx, removeCollectorQuery, arg.Removedversion, arg.Queryid, arg.Jobid)
return err
}
const setActiveCollectorVersion = `-- name: SetActiveCollectorVersion :exec
INSERT INTO collectorActiveVersions (collectorId, versionId) VALUES ($1, $2)
INSERT INTO collectorActiveVersions (jobId, versionId) VALUES ($1, $2)
`
type SetActiveCollectorVersionParams struct {
Collectorid pgtype.UUID `db:"collectorid"`
Versionid int32 `db:"versionid"`
Jobid pgtype.UUID `db:"jobid"`
Versionid int32 `db:"versionid"`
}
// SetActiveCollectorVersion
//
// INSERT INTO collectorActiveVersions (collectorId, versionId) VALUES ($1, $2)
// INSERT INTO collectorActiveVersions (jobId, versionId) VALUES ($1, $2)
func (q *Queries) SetActiveCollectorVersion(ctx context.Context, arg *SetActiveCollectorVersionParams) error {
_, err := q.db.Exec(ctx, setActiveCollectorVersion, arg.Collectorid, arg.Versionid)
_, err := q.db.Exec(ctx, setActiveCollectorVersion, arg.Jobid, arg.Versionid)
return err
}
const setCollectorCleanVersion = `-- name: SetCollectorCleanVersion :exec
INSERT INTO collectorMinCleanVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3)
INSERT INTO collectorMinCleanVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3)
`
type SetCollectorCleanVersionParams struct {
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Addedversion int32 `db:"addedversion"`
Versionid int32 `db:"versionid"`
}
// SetCollectorCleanVersion
//
// INSERT INTO collectorMinCleanVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3)
// INSERT INTO collectorMinCleanVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3)
func (q *Queries) SetCollectorCleanVersion(ctx context.Context, arg *SetCollectorCleanVersionParams) error {
_, err := q.db.Exec(ctx, setCollectorCleanVersion, arg.Collectorid, arg.Addedversion, arg.Versionid)
_, err := q.db.Exec(ctx, setCollectorCleanVersion, arg.Jobid, arg.Addedversion, arg.Versionid)
return err
}
const setCollectorTextVersion = `-- name: SetCollectorTextVersion :exec
INSERT INTO collectorMinTextVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3)
INSERT INTO collectorMinTextVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3)
`
type SetCollectorTextVersionParams struct {
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Addedversion int32 `db:"addedversion"`
Versionid int32 `db:"versionid"`
}
// SetCollectorTextVersion
//
// INSERT INTO collectorMinTextVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3)
// INSERT INTO collectorMinTextVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3)
func (q *Queries) SetCollectorTextVersion(ctx context.Context, arg *SetCollectorTextVersionParams) error {
_, err := q.db.Exec(ctx, setCollectorTextVersion, arg.Collectorid, arg.Addedversion, arg.Versionid)
_, err := q.db.Exec(ctx, setCollectorTextVersion, arg.Jobid, arg.Addedversion, arg.Versionid)
return err
}
internal/database/repository/collector_test.go
+27 -35
View File
@@ -38,11 +38,16 @@ func TestCollector(t *testing.T) {
assert.NoError(t, err)
version, err := queries.AddLatestQueryVersion(ctx, jsonId)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: jsonId,
Versionid: version,
})
assert.NoError(t, err)
assert.Equal(t, int32(1), version)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: jsonId,
Requiredqueryid: contextId,
Addedversion: 1,
Addedversion: version,
})
assert.NoError(t, err)
@@ -53,13 +58,9 @@ func TestCollector(t *testing.T) {
minCleanVersion := int32(2)
minTextVersion := int32(4)
collId, err := queries.CreateCollector(ctx, jobId)
assert.NoError(t, err)
coll, err := queries.GetCollector(ctx, collId)
coll, err := queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: 0,
Mintextversion: 0,
@@ -70,7 +71,6 @@ func TestCollector(t *testing.T) {
coll, err = queries.GetCollectorByJobID(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: 0,
Mintextversion: 0,
@@ -78,14 +78,13 @@ func TestCollector(t *testing.T) {
Latestversion: 0,
}, coll)
version, err = queries.AddLatestCollectorVersion(ctx, collId)
version, err = queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
assert.Equal(t, int32(1), version)
coll, err = queries.GetCollector(ctx, collId)
coll, err = queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: 0,
Mintextversion: 0,
@@ -94,15 +93,14 @@ func TestCollector(t *testing.T) {
}, coll)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Versionid: 1,
Collectorid: collId,
Versionid: 1,
Jobid: jobId,
})
assert.NoError(t, err)
coll, err = queries.GetCollector(ctx, collId)
coll, err = queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: 0,
Mintextversion: 0,
@@ -111,7 +109,7 @@ func TestCollector(t *testing.T) {
}, coll)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collId,
Jobid: jobId,
Queryid: jsonId,
Addedversion: 1,
Name: "example_key",
@@ -119,16 +117,15 @@ func TestCollector(t *testing.T) {
assert.NoError(t, err)
err = queries.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{
Collectorid: collId,
Jobid: jobId,
Addedversion: 1,
Versionid: minTextVersion,
})
assert.NoError(t, err)
coll, err = queries.GetCollector(ctx, collId)
coll, err = queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: 0,
Mintextversion: minTextVersion,
@@ -138,16 +135,15 @@ func TestCollector(t *testing.T) {
}, coll)
err = queries.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{
Collectorid: collId,
Jobid: jobId,
Addedversion: 1,
Versionid: minCleanVersion,
})
assert.NoError(t, err)
coll, err = queries.GetCollector(ctx, collId)
coll, err = queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
@@ -156,20 +152,18 @@ func TestCollector(t *testing.T) {
Fields: []byte(fmt.Sprintf("{\"example_key\": \"%s\"}", database.MustToUUID(jsonId).String())),
}, coll)
qs, err := queries.ListCollectorQueries(ctx, collId)
qs, err := queries.ListCollectorQueries(ctx, jobId)
assert.NoError(t, err)
assert.Len(t, qs, 2)
assert.ElementsMatch(t, []*repository.Collectorquerydependencytree{
{
Collectorid: collId,
Jobid: jobId,
Queryid: jsonId,
Queryversion: 0,
Queryversion: 1,
Type: repository.QuerytypeJsonExtractor,
Requiredids: []pgtype.UUID{contextId},
},
{
Collectorid: collId,
Jobid: jobId,
Queryid: contextId,
Queryversion: 0,
@@ -178,34 +172,33 @@ func TestCollector(t *testing.T) {
},
}, qs)
version, err = queries.AddLatestCollectorVersion(ctx, collId)
version, err = queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
assert.Equal(t, int32(2), version)
err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
Collectorid: collId,
Jobid: jobId,
Queryid: jsonId,
Removedversion: &version,
})
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Versionid: 2,
Collectorid: collId,
Versionid: 2,
Jobid: jobId,
})
assert.NoError(t, err)
err = queries.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{
Collectorid: collId,
Jobid: jobId,
Versionid: minTextVersion + 1,
Addedversion: version,
})
assert.NoError(t, err)
coll, err = queries.GetCollector(ctx, collId)
coll, err = queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion + 1,
@@ -215,16 +208,15 @@ func TestCollector(t *testing.T) {
}, coll)
err = queries.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{
Collectorid: collId,
Jobid: jobId,
Versionid: minCleanVersion + 1,
Addedversion: version,
})
assert.NoError(t, err)
coll, err = queries.GetCollector(ctx, collId)
coll, err = queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion + 1,
Mintextversion: minTextVersion + 1,
internal/database/repository/job.sql.go
+166 -18
View File
@@ -43,15 +43,9 @@ func (q *Queries) CreateJob(ctx context.Context, clientid pgtype.UUID) (pgtype.U
}
const getJob = `-- name: GetJob :one
SELECT j.id, j.clientId, coalesce(cs.canSync, false) as canSync
SELECT j.id, j.clientId, cs.canSync
FROM jobs as j
LEFT JOIN (
SELECT jobId, canSync
FROM jobCanSync
WHERE jobId = $1
ORDER BY id DESC
LIMIT 1
) as cs on cs.jobId = j.id
JOIN currentJobCanSync as cs on cs.jobId = j.id
WHERE j.id = $1
`
@@ -63,23 +57,177 @@ type GetJobRow struct {
// GetJob
//
// SELECT j.id, j.clientId, coalesce(cs.canSync, false) as canSync
// SELECT j.id, j.clientId, cs.canSync
// FROM jobs as j
// LEFT JOIN (
// SELECT jobId, canSync
// FROM jobCanSync
// WHERE jobId = $1
// ORDER BY id DESC
// LIMIT 1
// ) as cs on cs.jobId = j.id
// JOIN currentJobCanSync as cs on cs.jobId = j.id
// WHERE j.id = $1
func (q *Queries) GetJob(ctx context.Context, jobid pgtype.UUID) (*GetJobRow, error) {
row := q.db.QueryRow(ctx, getJob, jobid)
func (q *Queries) GetJob(ctx context.Context, id pgtype.UUID) (*GetJobRow, error) {
row := q.db.QueryRow(ctx, getJob, id)
var i GetJobRow
err := row.Scan(&i.ID, &i.Clientid, &i.Cansync)
return &i, err
}
const isJobSynced = `-- name: IsJobSynced :one
WITH
docs AS (
-- Get all documents for this job
SELECT id, jobId FROM documents WHERE jobId = $1
),
doc_text_entries AS (
-- Documents with their current text entries
SELECT
d.id AS document_id,
cte.id AS text_entry_id
FROM
docs d
LEFT JOIN currentTextEntries cte ON cte.documentId = d.id
),
required_results AS (
-- All required document-query-version combinations
SELECT
d.id AS document_id,
cqdt.queryId,
cqdt.queryVersion,
dte.text_entry_id
FROM
docs d
JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.jobId
JOIN doc_text_entries dte ON dte.document_id = d.id
and dte.text_entry_id IS NOT NULL
),
existing_results AS (
-- Valid results that exist
SELECT
rr.document_id AS document_id,
r.queryId,
r.id AS result_id
FROM
results r
JOIN required_results rr ON
r.queryId = rr.queryId AND
r.queryVersion = rr.queryVersion AND
r.textEntryId = rr.text_entry_id
),
missing_results AS (
-- Find missing results
SELECT rr.queryId
FROM required_results rr
LEFT JOIN existing_results er on er.queryId = rr.queryId
WHERE er.result_id is null
),
dependency_check AS (
-- Check for missing dependencies
SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId
FROM existing_results er
JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId
LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id
WHERE rd.resultId is null
)
SELECT (
-- No documents means job is synced
NOT EXISTS (SELECT 1 FROM docs)
OR
-- Documents with no text entries
(NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL)
and
-- Documents with missing results
NOT EXISTS (SELECT 1 FROM missing_results)
and
-- Documents with missing dependencies
NOT EXISTS (SELECT 1 FROM dependency_check))
)::bool as is_synced
`
// IsJobSynced
//
// WITH
// docs AS (
// -- Get all documents for this job
// SELECT id, jobId FROM documents WHERE jobId = $1
// ),
// doc_text_entries AS (
// -- Documents with their current text entries
// SELECT
// d.id AS document_id,
// cte.id AS text_entry_id
// FROM
// docs d
// LEFT JOIN currentTextEntries cte ON cte.documentId = d.id
// ),
// required_results AS (
// -- All required document-query-version combinations
// SELECT
// d.id AS document_id,
// cqdt.queryId,
// cqdt.queryVersion,
// dte.text_entry_id
// FROM
// docs d
// JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.jobId
// JOIN doc_text_entries dte ON dte.document_id = d.id
// and dte.text_entry_id IS NOT NULL
// ),
// existing_results AS (
// -- Valid results that exist
// SELECT
// rr.document_id AS document_id,
// r.queryId,
// r.id AS result_id
// FROM
// results r
// JOIN required_results rr ON
// r.queryId = rr.queryId AND
// r.queryVersion = rr.queryVersion AND
// r.textEntryId = rr.text_entry_id
// ),
// missing_results AS (
// -- Find missing results
// SELECT rr.queryId
// FROM required_results rr
// LEFT JOIN existing_results er on er.queryId = rr.queryId
// WHERE er.result_id is null
// ),
// dependency_check AS (
// -- Check for missing dependencies
// SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId
// FROM existing_results er
// JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId
// LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id
// WHERE rd.resultId is null
// )
// SELECT (
// -- No documents means job is synced
// NOT EXISTS (SELECT 1 FROM docs)
//
// OR
//
// -- Documents with no text entries
// (NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL)
//
// and
//
// -- Documents with missing results
// NOT EXISTS (SELECT 1 FROM missing_results)
//
// and
//
// -- Documents with missing dependencies
// NOT EXISTS (SELECT 1 FROM dependency_check))
// )::bool as is_synced
func (q *Queries) IsJobSynced(ctx context.Context, dollar_1 pgtype.UUID) (bool, error) {
row := q.db.QueryRow(ctx, isJobSynced, dollar_1)
var is_synced bool
err := row.Scan(&is_synced)
return is_synced, err
}
const listJobDocumentIDsBatch = `-- name: ListJobDocumentIDsBatch :many
SELECT id, totalCount FROM listJobDocumentIDs($1, $2, $3)
`
internal/database/repository/job_test.go
+425
View File
@@ -164,3 +164,428 @@ func TestListJobDocumentIDs(t *testing.T) {
},
}, ids)
}
func TestJobSync(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../.."))
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Cfg: cfg,
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, contextQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: contextQueryID,
Versionid: 1,
})
assert.NoError(t, err)
jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, jsonQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: jsonQueryID,
Versionid: 1,
})
assert.NoError(t, err)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: jsonQueryID,
Requiredqueryid: contextQueryID,
Addedversion: 1,
})
assert.NoError(t, err)
clientId, err := queries.CreateClient(ctx, "example_client")
assert.NoError(t, err)
jobId, err := queries.CreateJob(ctx, clientId)
assert.NoError(t, err)
_, err = queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Versionid: 1,
Jobid: jobId,
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobId,
Queryid: jsonQueryID,
Addedversion: 1,
Name: "example_key",
})
assert.NoError(t, err)
isSynced, err := queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Jobid: jobId,
Hash: "example_hash",
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID)
assert.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
textentry, err := queries.GetDocumentTextEntry(ctx, documentID)
assert.NoError(t, err)
depresultid, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
resultid, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: resultid,
Requiredresultid: depresultid,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
_, err = queries.AddLatestQueryVersion(ctx, contextQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: contextQueryID,
Versionid: 2,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 2,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
resultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: resultid,
Requiredresultid: depresultid,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
textentry, err = queries.GetDocumentTextEntry(ctx, documentID)
assert.NoError(t, err)
depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 2,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
resultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: resultid,
Requiredresultid: depresultid,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
cleanentry, err = queries.GetDocumentCleanEntry(ctx, documentID)
assert.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
textentry, err = queries.GetDocumentTextEntry(ctx, documentID)
assert.NoError(t, err)
depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 2,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
resultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: resultid,
Requiredresultid: depresultid,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
_, err = queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
collversion := int32(2)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Versionid: 2,
Jobid: jobId,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
Jobid: jobId,
Queryid: jsonQueryID,
Removedversion: &collversion,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobId,
Queryid: jsonQueryID,
Addedversion: 2,
Name: "second_key",
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobId,
Queryid: contextQueryID,
Addedversion: 2,
Name: "example_key",
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
superQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, superQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: superQueryID,
Versionid: 1,
})
assert.NoError(t, err)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: contextQueryID,
Requiredqueryid: superQueryID,
Addedversion: 1,
})
assert.NoError(t, err)
superresultid, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: superQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 2,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
resultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: resultid,
Requiredresultid: depresultid,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: depresultid,
Requiredresultid: superresultid,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
}
internal/database/repository/models.go
+90 -31
View File
@@ -73,30 +73,25 @@ type Clientcansync struct {
Cansync bool `db:"cansync"`
}
type Collector struct {
ID pgtype.UUID `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
}
type Collectoractiveversion struct {
ID pgtype.UUID `db:"id"`
Collectorid pgtype.UUID `db:"collectorid"`
Versionid int32 `db:"versionid"`
ID pgtype.UUID `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
Versionid int32 `db:"versionid"`
}
type Collectorcurrentactiveversion struct {
Collectorid pgtype.UUID `db:"collectorid"`
ID int32 `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
Activeversion int32 `db:"activeversion"`
}
type Collectorlatestversion struct {
Collectorid pgtype.UUID `db:"collectorid"`
ID int32 `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
Latestversion int32 `db:"latestversion"`
}
type Collectormincleanversion struct {
ID pgtype.UUID `db:"id"`
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Versionid int32 `db:"versionid"`
Addedversion int32 `db:"addedversion"`
Removedversion *int32 `db:"removedversion"`
@@ -104,7 +99,7 @@ type Collectormincleanversion struct {
type Collectormintextversion struct {
ID pgtype.UUID `db:"id"`
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Versionid int32 `db:"versionid"`
Addedversion int32 `db:"addedversion"`
Removedversion *int32 `db:"removedversion"`
@@ -112,7 +107,7 @@ type Collectormintextversion struct {
type Collectorquery struct {
ID pgtype.UUID `db:"id"`
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Name string `db:"name"`
Queryid pgtype.UUID `db:"queryid"`
Addedversion int32 `db:"addedversion"`
@@ -120,7 +115,6 @@ type Collectorquery struct {
}
type Collectorquerydependencytree struct {
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Queryid pgtype.UUID `db:"queryid"`
Type Querytype `db:"type"`
@@ -129,9 +123,57 @@ type Collectorquerydependencytree struct {
}
type Collectorversion struct {
Collectorid pgtype.UUID `db:"collectorid"`
ID int32 `db:"id"`
Addedat pgtype.Timestamp `db:"addedat"`
Jobid pgtype.UUID `db:"jobid"`
ID int32 `db:"id"`
Addedat pgtype.Timestamp `db:"addedat"`
}
type Currentcleanentry struct {
ID pgtype.UUID `db:"id"`
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
}
type Currentclientcansync struct {
Clientid pgtype.UUID `db:"clientid"`
Cansync bool `db:"cansync"`
}
type Currentcollectormincleanversion struct {
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
}
type Currentcollectormintextversion struct {
Jobid pgtype.UUID `db:"jobid"`
Mintextversion int32 `db:"mintextversion"`
}
type Currentcollectorqueriesjsonagg struct {
Jobid pgtype.UUID `db:"jobid"`
Fields []byte `db:"fields"`
}
type Currentcollectorquery struct {
Jobid pgtype.UUID `db:"jobid"`
Name *string `db:"name"`
Queryid pgtype.UUID `db:"queryid"`
}
type Currentjobcansync struct {
Jobid pgtype.UUID `db:"jobid"`
Cansync bool `db:"cansync"`
}
type Currenttextentry struct {
ID pgtype.UUID `db:"id"`
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
Cleanentryid pgtype.UUID `db:"cleanentryid"`
}
type Document struct {
@@ -156,15 +198,14 @@ type Documententry struct {
}
type Documenttextextraction struct {
ID pgtype.UUID `db:"id"`
Documentid pgtype.UUID `db:"documentid"`
Version int32 `db:"version"`
Bucket string `db:"bucket"`
Key string `db:"key"`
ID pgtype.UUID `db:"id"`
Cleanentryid pgtype.UUID `db:"cleanentryid"`
Version int32 `db:"version"`
Bucket string `db:"bucket"`
Key string `db:"key"`
}
type Fullactivecollector struct {
ID pgtype.UUID `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
@@ -218,13 +259,28 @@ type Queryconfig struct {
}
type Querycurrentactiveversion struct {
Queryid pgtype.UUID `db:"queryid"`
Activeversion int32 `db:"activeversion"`
}
type Querycurrentconfig struct {
Queryid pgtype.UUID `db:"queryid"`
ID int32 `db:"id"`
Config []byte `db:"config"`
}
type Querycurrentrequiredid struct {
Queryid pgtype.UUID `db:"queryid"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
}
type Querycurrentrequiredidsagg struct {
Queryid pgtype.UUID `db:"queryid"`
Requiredids []pgtype.UUID `db:"requiredids"`
}
type Querylatestversion struct {
Queryid pgtype.UUID `db:"queryid"`
ID int32 `db:"id"`
Queryid pgtype.UUID `db:"queryid"`
Latestversion int32 `db:"latestversion"`
}
type Queryversion struct {
@@ -243,10 +299,13 @@ type Requiredquery struct {
type Result struct {
ID pgtype.UUID `db:"id"`
Textentryid pgtype.UUID `db:"textentryid"`
Queryid pgtype.UUID `db:"queryid"`
Documentid pgtype.UUID `db:"documentid"`
Value string `db:"value"`
Cleanversion int32 `db:"cleanversion"`
Textversion int32 `db:"textversion"`
Queryversion int32 `db:"queryversion"`
}
type Resultdependency struct {
Resultid pgtype.UUID `db:"resultid"`
Requiredresultid pgtype.UUID `db:"requiredresultid"`
}
internal/database/repository/query.sql.go
+56 -56
View File
@@ -92,6 +92,20 @@ func (q *Queries) CreateQuery(ctx context.Context, type_ Querytype) (pgtype.UUID
return id, err
}
const getActiveQueryConfig = `-- name: GetActiveQueryConfig :one
SELECT config FROM queryCurrentConfigs where queryId = $1
`
// GetActiveQueryConfig
//
// SELECT config FROM queryCurrentConfigs where queryId = $1
func (q *Queries) GetActiveQueryConfig(ctx context.Context, queryid pgtype.UUID) ([]byte, error) {
row := q.db.QueryRow(ctx, getActiveQueryConfig, queryid)
var config []byte
err := row.Scan(&config)
return config, err
}
const getQuery = `-- name: GetQuery :one
SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = $1
`
@@ -113,30 +127,6 @@ func (q *Queries) GetQuery(ctx context.Context, id pgtype.UUID) (*Fullactivequer
return &i, err
}
const getQueryConfig = `-- name: GetQueryConfig :one
SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 and COALESCE(removedVersion, $2 - 1) < $2
`
type GetQueryConfigParams struct {
Queryid pgtype.UUID `db:"queryid"`
Addedversion int32 `db:"addedversion"`
}
type GetQueryConfigRow struct {
ID pgtype.UUID `db:"id"`
Config []byte `db:"config"`
}
// GetQueryConfig
//
// SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 and COALESCE(removedVersion, $2 - 1) < $2
func (q *Queries) GetQueryConfig(ctx context.Context, arg *GetQueryConfigParams) (*GetQueryConfigRow, error) {
row := q.db.QueryRow(ctx, getQueryConfig, arg.Queryid, arg.Addedversion)
var i GetQueryConfigRow
err := row.Scan(&i.ID, &i.Config)
return &i, err
}
const getQueryWithVersion = `-- name: GetQueryWithVersion :one
WITH query as (
SELECT id, type FROM queries WHERE id = $1
@@ -149,22 +139,23 @@ SELECT c.queryId, c.config
),
requiredIds as (
SELECT r.queryId,
coalesce(
ARRAY_AGG(DISTINCT r.requiredQueryId)
FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[]
as requiredIds
FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
array[]::uuid[]
)::uuid[] as requiredIds
FROM query AS q
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
and isInVersion($2, r.addedVersion, r.removedVersion)
GROUP BY r.queryId
)
SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config,
coalesce(
r.requiredIds,
array[]::uuid[]
)::uuid[] as requiredIds
SELECT DISTINCT q.id, q.type,
av.activeVersion,
lv.latestVersion, c.config,
r.requiredIds
FROM query AS q
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id
JOIN queryCurrentActiveVersions as av on q.id = av.queryId
JOIN queryLatestVersions as lv on lv.queryId = q.id
LEFT JOIN config AS c ON q.id = c.queryId
LEFT JOIN requiredIds AS r ON q.id = r.queryId
`
@@ -196,22 +187,23 @@ type GetQueryWithVersionRow struct {
// ),
// requiredIds as (
// SELECT r.queryId,
// coalesce(
// ARRAY_AGG(DISTINCT r.requiredQueryId)
// FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[]
// as requiredIds
// FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
// array[]::uuid[]
// )::uuid[] as requiredIds
// FROM query AS q
// LEFT JOIN requiredQueries AS r ON q.id = r.queryId
// and isInVersion($2, r.addedVersion, r.removedVersion)
// GROUP BY r.queryId
// )
// SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config,
// coalesce(
// r.requiredIds,
// array[]::uuid[]
// )::uuid[] as requiredIds
// SELECT DISTINCT q.id, q.type,
// av.activeVersion,
// lv.latestVersion, c.config,
// r.requiredIds
// FROM query AS q
// LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
// LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id
// JOIN queryCurrentActiveVersions as av on q.id = av.queryId
// JOIN queryLatestVersions as lv on lv.queryId = q.id
// LEFT JOIN config AS c ON q.id = c.queryId
// LEFT JOIN requiredIds AS r ON q.id = r.queryId
func (q *Queries) GetQueryWithVersion(ctx context.Context, arg *GetQueryWithVersionParams) (*GetQueryWithVersionRow, error) {
@@ -230,22 +222,28 @@ func (q *Queries) GetQueryWithVersion(ctx context.Context, arg *GetQueryWithVers
const isQueryInDependencyTree = `-- name: IsQueryInDependencyTree :one
SELECT EXISTS (
SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1)
SELECT 1 FROM queryActiveDependencies
WHERE id = any($1)
and requiredQueryId = $2
or $2 = any($1)
)
`
type IsQueryInDependencyTreeParams struct {
ID []pgtype.UUID `db:"id"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
Requiredqueryids []pgtype.UUID `db:"requiredqueryids"`
Queryid pgtype.UUID `db:"queryid"`
}
// IsQueryInDependencyTree
//
// SELECT EXISTS (
// SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1)
// SELECT 1 FROM queryActiveDependencies
// WHERE id = any($1)
// and requiredQueryId = $2
// or $2 = any($1)
// )
func (q *Queries) IsQueryInDependencyTree(ctx context.Context, arg *IsQueryInDependencyTreeParams) (bool, error) {
row := q.db.QueryRow(ctx, isQueryInDependencyTree, arg.ID, arg.Requiredqueryid)
row := q.db.QueryRow(ctx, isQueryInDependencyTree, arg.Requiredqueryids, arg.Queryid)
var exists bool
err := row.Scan(&exists)
return exists, err
@@ -324,14 +322,15 @@ WITH doc AS (
SELECT id, jobId FROM documents where id = $2
)
SELECT dt.queryId
FROM collectorQueryDependencyTree as dt
JOIN doc as d on d.jobId = dt.jobId
where $1 = any(dt.requiredIds)
FROM doc as d
JOIN collectorQueryDependencyTree as dt
on d.jobId = dt.jobId
and $1 = any(dt.requiredIds)
`
type ListQueryDirectDependentsByDocumentIDParams struct {
Requiredids pgtype.UUID `db:"requiredids"`
ID pgtype.UUID `db:"id"`
Queryid pgtype.UUID `db:"queryid"`
Documentid pgtype.UUID `db:"documentid"`
}
// ListQueryDirectDependentsByDocumentID
@@ -340,11 +339,12 @@ type ListQueryDirectDependentsByDocumentIDParams struct {
// SELECT id, jobId FROM documents where id = $2
// )
// SELECT dt.queryId
// FROM collectorQueryDependencyTree as dt
// JOIN doc as d on d.jobId = dt.jobId
// where $1 = any(dt.requiredIds)
// FROM doc as d
// JOIN collectorQueryDependencyTree as dt
// on d.jobId = dt.jobId
// and $1 = any(dt.requiredIds)
func (q *Queries) ListQueryDirectDependentsByDocumentID(ctx context.Context, arg *ListQueryDirectDependentsByDocumentIDParams) ([]pgtype.UUID, error) {
rows, err := q.db.Query(ctx, listQueryDirectDependentsByDocumentID, arg.Requiredids, arg.ID)
rows, err := q.db.Query(ctx, listQueryDirectDependentsByDocumentID, arg.Queryid, arg.Documentid)
if err != nil {
return nil, err
}
internal/database/repository/query_test.go
+95 -82
View File
@@ -152,13 +152,6 @@ func TestQueries(t *testing.T) {
})
assert.NoError(t, err)
err = queries.SetQueryConfig(ctx, &repository.SetQueryConfigParams{
Queryid: jsonQueryID,
Config: []byte(`{"second":"key"}`),
Addedversion: 2,
})
assert.NoError(t, err)
err = queries.SetQueryConfig(ctx, &repository.SetQueryConfigParams{
Queryid: jsonQueryID,
Config: jsonConfig,
@@ -166,29 +159,40 @@ func TestQueries(t *testing.T) {
})
assert.NoError(t, err)
jsonQueryConfig, err := queries.GetQueryConfig(ctx, &repository.GetQueryConfigParams{
Queryid: jsonQueryID,
Addedversion: jsonQuery.Activeversion,
})
jsonQueryConfig, err := queries.GetActiveQueryConfig(ctx, jsonQueryID)
assert.NoError(t, err)
assert.Equal(t, jsonConfig, jsonQueryConfig.Config)
assert.Equal(t, jsonConfig, jsonQueryConfig)
jsonQueryConfig, err = queries.GetQueryConfig(ctx, &repository.GetQueryConfigParams{
err = queries.SetQueryConfig(ctx, &repository.SetQueryConfigParams{
Queryid: jsonQueryID,
Config: []byte(`{"second":"key"}`),
Addedversion: 2,
})
assert.NoError(t, err)
assert.Equal(t, []byte(`{"second": "key"}`), jsonQueryConfig.Config)
jsonQueryConfig, err = queries.GetActiveQueryConfig(ctx, jsonQueryID)
assert.NoError(t, err)
assert.Equal(t, jsonConfig, jsonQueryConfig)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: jsonQueryID,
Versionid: 2,
})
assert.NoError(t, err)
jsonQueryConfig, err = queries.GetActiveQueryConfig(ctx, jsonQueryID)
assert.NoError(t, err)
assert.Equal(t, []byte(`{"second": "key"}`), jsonQueryConfig)
jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivequery{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Activeversion: 2,
Latestversion: 2,
Config: jsonConfig,
Requiredids: []pgtype.UUID{contextQueryID},
Config: jsonQueryConfig,
Requiredids: []pgtype.UUID{},
}, jsonQuery)
v := int32(1)
@@ -200,7 +204,7 @@ func TestQueries(t *testing.T) {
assert.EqualExportedValues(t, &repository.GetQueryWithVersionRow{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Activeversion: 2,
Latestversion: 2,
Config: jsonConfig,
Requiredids: []pgtype.UUID{contextQueryID},
@@ -248,13 +252,11 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
jobID, err := queries.CreateJob(ctx, clientID)
assert.NoError(t, err)
collID, err := queries.CreateCollector(ctx, jobID)
assert.NoError(t, err)
_, err = queries.AddLatestCollectorVersion(ctx, collID)
version, err := queries.AddLatestCollectorVersion(ctx, jobID)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Collectorid: collID,
Versionid: 1,
Jobid: jobID,
Versionid: version,
})
assert.NoError(t, err)
docID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
@@ -265,10 +267,17 @@ func TestQueryDependencyTree(t *testing.T) {
contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, contextQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: contextQueryID,
Versionid: 1,
})
assert.NoError(t, err)
dependents, err := queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
@@ -277,10 +286,15 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, jsonQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: jsonQueryID,
Versionid: 1,
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
@@ -293,20 +307,20 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collID,
Jobid: jobID,
Name: "example_name'",
Queryid: jsonQueryID,
Addedversion: 1,
@@ -314,14 +328,14 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
@@ -330,16 +344,21 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, secondJsonQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: secondJsonQueryID,
Versionid: 1,
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
@@ -352,26 +371,26 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: secondJsonQueryID,
Documentid: docID,
Queryid: secondJsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collID,
Jobid: jobID,
Name: "example_name'",
Queryid: secondJsonQueryID,
Addedversion: 1,
@@ -379,55 +398,55 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: secondJsonQueryID,
Documentid: docID,
Queryid: secondJsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{secondJsonQueryID}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
isdependent, err := queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: jsonQueryID,
ID: []pgtype.UUID{contextQueryID},
Queryid: jsonQueryID,
Requiredqueryids: []pgtype.UUID{contextQueryID},
})
assert.NoError(t, err)
assert.False(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: jsonQueryID,
ID: []pgtype.UUID{secondJsonQueryID},
Queryid: jsonQueryID,
Requiredqueryids: []pgtype.UUID{secondJsonQueryID},
})
assert.NoError(t, err)
assert.True(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: jsonQueryID,
ID: []pgtype.UUID{jsonQueryID},
Queryid: jsonQueryID,
Requiredqueryids: []pgtype.UUID{jsonQueryID},
})
assert.NoError(t, err)
assert.True(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: secondJsonQueryID,
ID: []pgtype.UUID{jsonQueryID, contextQueryID},
Queryid: secondJsonQueryID,
Requiredqueryids: []pgtype.UUID{jsonQueryID, contextQueryID},
})
assert.NoError(t, err)
assert.False(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: contextQueryID,
ID: []pgtype.UUID{jsonQueryID, secondJsonQueryID},
Queryid: contextQueryID,
Requiredqueryids: []pgtype.UUID{jsonQueryID, secondJsonQueryID},
})
assert.NoError(t, err)
assert.True(t, isdependent)
@@ -521,17 +540,15 @@ func TestListQueryJobs(t *testing.T) {
assert.NoError(t, err)
jobOneID, err := queries.CreateJob(ctx, clientOneID)
assert.NoError(t, err)
collOneID, err := queries.CreateCollector(ctx, jobOneID)
assert.NoError(t, err)
_, err = queries.AddLatestCollectorVersion(ctx, collOneID)
versionOne, err := queries.AddLatestCollectorVersion(ctx, jobOneID)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Collectorid: collOneID,
Versionid: 1,
Jobid: jobOneID,
Versionid: versionOne,
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collOneID,
Jobid: jobOneID,
Queryid: contextID,
Addedversion: 1,
Name: "example_key",
@@ -544,17 +561,15 @@ func TestListQueryJobs(t *testing.T) {
jobTwoID, err := queries.CreateJob(ctx, clientOneID)
assert.NoError(t, err)
collTwoID, err := queries.CreateCollector(ctx, jobTwoID)
assert.NoError(t, err)
_, err = queries.AddLatestCollectorVersion(ctx, collTwoID)
versionTwo, err := queries.AddLatestCollectorVersion(ctx, jobTwoID)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Collectorid: collTwoID,
Versionid: 1,
Jobid: jobTwoID,
Versionid: versionTwo,
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collTwoID,
Jobid: jobTwoID,
Queryid: contextID,
Addedversion: 1,
Name: "example_key",
@@ -569,17 +584,15 @@ func TestListQueryJobs(t *testing.T) {
assert.NoError(t, err)
jobThreeID, err := queries.CreateJob(ctx, clientTwoID)
assert.NoError(t, err)
collThreeID, err := queries.CreateCollector(ctx, jobThreeID)
assert.NoError(t, err)
_, err = queries.AddLatestCollectorVersion(ctx, collThreeID)
versionThree, err := queries.AddLatestCollectorVersion(ctx, jobThreeID)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Collectorid: collThreeID,
Versionid: 1,
Jobid: jobThreeID,
Versionid: versionThree,
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collThreeID,
Jobid: jobThreeID,
Queryid: contextID,
Addedversion: 1,
Name: "example_key",
@@ -601,7 +614,7 @@ func TestListQueryJobs(t *testing.T) {
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collOneID,
Jobid: jobOneID,
Queryid: jsonID,
Addedversion: 1,
Name: "example_key",
internal/database/repository/result.sql.go
+133 -120
View File
@@ -11,34 +11,91 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
const addResult = `-- name: AddResult :one
INSERT INTO results (queryId, value, textEntryId, queryVersion) VALUES ($1, $2, $3, $4) returning id
`
type AddResultParams struct {
Queryid pgtype.UUID `db:"queryid"`
Value string `db:"value"`
Textentryid pgtype.UUID `db:"textentryid"`
Queryversion int32 `db:"queryversion"`
}
// AddResult
//
// INSERT INTO results (queryId, value, textEntryId, queryVersion) VALUES ($1, $2, $3, $4) returning id
func (q *Queries) AddResult(ctx context.Context, arg *AddResultParams) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, addResult,
arg.Queryid,
arg.Value,
arg.Textentryid,
arg.Queryversion,
)
var id pgtype.UUID
err := row.Scan(&id)
return id, err
}
const addResultDependency = `-- name: AddResultDependency :exec
INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2)
`
type AddResultDependencyParams struct {
Resultid pgtype.UUID `db:"resultid"`
Requiredresultid pgtype.UUID `db:"requiredresultid"`
}
// AddResultDependency
//
// INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2)
func (q *Queries) AddResultDependency(ctx context.Context, arg *AddResultDependencyParams) error {
_, err := q.db.Exec(ctx, addResultDependency, arg.Resultid, arg.Requiredresultid)
return err
}
const getResultValueWithVersion = `-- name: GetResultValueWithVersion :one
SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5
WITH doc as (
SELECT id, jobId
FROM documents
WHERE id = $3
)
SELECT r.id, r.value
FROM doc as d
JOIN currentTextEntries as cte on cte.documentId = d.id
LEFT JOIN results as r
on r.queryId = $1
and r.queryVersion = $2
and r.textEntryId = cte.id
`
type GetResultValueWithVersionParams struct {
Queryid pgtype.UUID `db:"queryid"`
Queryversion int32 `db:"queryversion"`
Queryversion *int32 `db:"queryversion"`
Documentid pgtype.UUID `db:"documentid"`
Cleanversion int32 `db:"cleanversion"`
Textversion int32 `db:"textversion"`
}
type GetResultValueWithVersionRow struct {
ID pgtype.UUID `db:"id"`
Value string `db:"value"`
Value *string `db:"value"`
}
// GetResultValueWithVersion
//
// SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5
// WITH doc as (
// SELECT id, jobId
// FROM documents
// WHERE id = $3
// )
// SELECT r.id, r.value
// FROM doc as d
// JOIN currentTextEntries as cte on cte.documentId = d.id
// LEFT JOIN results as r
// on r.queryId = $1
// and r.queryVersion = $2
// and r.textEntryId = cte.id
func (q *Queries) GetResultValueWithVersion(ctx context.Context, arg *GetResultValueWithVersionParams) (*GetResultValueWithVersionRow, error) {
row := q.db.QueryRow(ctx, getResultValueWithVersion,
arg.Queryid,
arg.Queryversion,
arg.Documentid,
arg.Cleanversion,
arg.Textversion,
)
row := q.db.QueryRow(ctx, getResultValueWithVersion, arg.Queryid, arg.Queryversion, arg.Documentid)
var i GetResultValueWithVersionRow
err := row.Scan(&i.ID, &i.Value)
return &i, err
@@ -46,131 +103,110 @@ func (q *Queries) GetResultValueWithVersion(ctx context.Context, arg *GetResultV
const listQueryRequirementValues = `-- name: ListQueryRequirementValues :many
WITH reqQueries as (
SELECT q.id as queryId, av.id as activeVersion, q.type
SELECT av.queryId, av.activeVersion, q.type
FROM requiredQueries as rq
JOIN queries as q on q.id = rq.requiredQueryId
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
WHERE rq.queryId = $2
and isInVersion($3, rq.addedVersion, rq.removedVersion)
JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId
JOIN queries as q on q.id = av.queryId
WHERE rq.queryId = $1
and isInVersion($2, rq.addedVersion, rq.removedVersion)
),
docs as (
SELECT id, jobId
FROM documents
WHERE id = $1
WHERE id = $3
),
codeVersions as (
SELECT
SELECT
d.id as documentId,
coalesce(ccv.versionId, 1) as minCleanVersion,
coalesce(ctv.versionId, 1) as minTextVersion
mcv.minCleanVersion,
mtv.minTextVersion
FROM docs as d
LEFT JOIN collectors as c on c.jobId = d.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId
and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion)
LIMIT 1
JOIN currentCollectorMinTextVersions as mtv on mtv.jobId = d.jobId
JOIN currentCollectorMinCleanVersions as mcv on mcv.jobId = d.jobId
),
latestVersions AS (
SELECT
SELECT
r.id,
rq.queryId,
rq.type,
r.queryVersion,
r.cleanVersion,
r.textVersion,
r.textEntryId,
r.value,
ROW_NUMBER() OVER (
PARTITION BY rq.queryId
ORDER BY r.cleanVersion DESC, r.textVersion DESC
PARTITION BY r.queryId
ORDER BY r.id DESC
) as rowNumber
FROM reqQueries as rq
JOIN currentTextEntries as cte on cte.documentId = $3
LEFT JOIN results as r ON rq.queryId = r.queryId
and r.documentId = $1
and r.queryVersion = rq.activeVersion
JOIN codeVersions as ccv on ccv.documentId = r.documentId
and r.cleanVersion >= ccv.minCleanVersion
and r.textVersion >= ccv.minTextVersion
and cte.id = r.textEntryId
)
SELECT DISTINCT lv.queryId, lv.type, r.value
FROM latestVersions as lv
JOIN results as r ON r.queryId = lv.queryId
and r.documentId = $1
and r.queryVersion = lv.queryVersion
and r.cleanVersion = lv.cleanVersion
and r.textVersion = lv.textVersion
and lv.rowNumber = 1
SELECT DISTINCT id, queryId, type, value
FROM latestVersions
WHERE rowNumber = 1
`
type ListQueryRequirementValuesParams struct {
Documentid pgtype.UUID `db:"documentid"`
Queryid pgtype.UUID `db:"queryid"`
Version int32 `db:"version"`
Version *int32 `db:"version"`
Documentid pgtype.UUID `db:"documentid"`
}
type ListQueryRequirementValuesRow struct {
ID pgtype.UUID `db:"id"`
Queryid pgtype.UUID `db:"queryid"`
Type Querytype `db:"type"`
Value string `db:"value"`
Value *string `db:"value"`
}
// ListQueryRequirementValues
//
// WITH reqQueries as (
// SELECT q.id as queryId, av.id as activeVersion, q.type
// SELECT av.queryId, av.activeVersion, q.type
// FROM requiredQueries as rq
// JOIN queries as q on q.id = rq.requiredQueryId
// LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
// WHERE rq.queryId = $2
// and isInVersion($3, rq.addedVersion, rq.removedVersion)
// JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId
// JOIN queries as q on q.id = av.queryId
// WHERE rq.queryId = $1
// and isInVersion($2, rq.addedVersion, rq.removedVersion)
// ),
// docs as (
// SELECT id, jobId
// FROM documents
// WHERE id = $1
// WHERE id = $3
// ),
// codeVersions as (
// SELECT
// d.id as documentId,
// coalesce(ccv.versionId, 1) as minCleanVersion,
// coalesce(ctv.versionId, 1) as minTextVersion
// mcv.minCleanVersion,
// mtv.minTextVersion
// FROM docs as d
// LEFT JOIN collectors as c on c.jobId = d.jobId
// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
// LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
// LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId
// and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion)
// LIMIT 1
// JOIN currentCollectorMinTextVersions as mtv on mtv.jobId = d.jobId
// JOIN currentCollectorMinCleanVersions as mcv on mcv.jobId = d.jobId
// ),
// latestVersions AS (
// SELECT
// r.id,
// rq.queryId,
// rq.type,
// r.queryVersion,
// r.cleanVersion,
// r.textVersion,
// r.textEntryId,
// r.value,
// ROW_NUMBER() OVER (
// PARTITION BY rq.queryId
// ORDER BY r.cleanVersion DESC, r.textVersion DESC
// PARTITION BY r.queryId
// ORDER BY r.id DESC
// ) as rowNumber
// FROM reqQueries as rq
// JOIN currentTextEntries as cte on cte.documentId = $3
// LEFT JOIN results as r ON rq.queryId = r.queryId
// and r.documentId = $1
// and r.queryVersion = rq.activeVersion
// JOIN codeVersions as ccv on ccv.documentId = r.documentId
// and r.cleanVersion >= ccv.minCleanVersion
// and r.textVersion >= ccv.minTextVersion
// and cte.id = r.textEntryId
// )
// SELECT DISTINCT lv.queryId, lv.type, r.value
// FROM latestVersions as lv
// JOIN results as r ON r.queryId = lv.queryId
// and r.documentId = $1
// and r.queryVersion = lv.queryVersion
// and r.cleanVersion = lv.cleanVersion
// and r.textVersion = lv.textVersion
// and lv.rowNumber = 1
// SELECT DISTINCT id, queryId, type, value
// FROM latestVersions
// WHERE rowNumber = 1
func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQueryRequirementValuesParams) ([]*ListQueryRequirementValuesRow, error) {
rows, err := q.db.Query(ctx, listQueryRequirementValues, arg.Documentid, arg.Queryid, arg.Version)
rows, err := q.db.Query(ctx, listQueryRequirementValues, arg.Queryid, arg.Version, arg.Documentid)
if err != nil {
return nil, err
}
@@ -178,7 +214,12 @@ func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQuery
items := []*ListQueryRequirementValuesRow{}
for rows.Next() {
var i ListQueryRequirementValuesRow
if err := rows.Scan(&i.Queryid, &i.Type, &i.Value); err != nil {
if err := rows.Scan(
&i.ID,
&i.Queryid,
&i.Type,
&i.Value,
); err != nil {
return nil, err
}
items = append(items, &i)
@@ -194,14 +235,14 @@ WITH docs as (
SELECT id, jobId from documents where id = $1
),
unsyncedQueries AS (
SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id
SELECT DISTINCT dt.queryId, dt.requiredIds
from docs as d
JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId
JOIN fullActiveCollectors as c on c.id = dt.collectorId
LEFT JOIN results as r on r.queryId = dt.queryId
and r.documentId = d.id
JOIN currentTextEntries as cte on cte.documentId = d.id
LEFT JOIN results as r
on r.queryId = dt.queryId
and r.queryVersion = dt.queryVersion
and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
and cte.id = r.textEntryId
where r.value is null
)
SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
@@ -216,14 +257,14 @@ SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
// SELECT id, jobId from documents where id = $1
// ),
// unsyncedQueries AS (
// SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id
// SELECT DISTINCT dt.queryId, dt.requiredIds
// from docs as d
// JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId
// JOIN fullActiveCollectors as c on c.id = dt.collectorId
// LEFT JOIN results as r on r.queryId = dt.queryId
// and r.documentId = d.id
// JOIN currentTextEntries as cte on cte.documentId = d.id
// LEFT JOIN results as r
// on r.queryId = dt.queryId
// and r.queryVersion = dt.queryVersion
// and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
// and cte.id = r.textEntryId
// where r.value is null
// )
// SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
@@ -249,31 +290,3 @@ func (q *Queries) ListUnsyncedNoDepsQueriesByDocId(ctx context.Context, dollar_1
}
return items, nil
}
const setResult = `-- name: SetResult :exec
INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6)
`
type SetResultParams struct {
Queryid pgtype.UUID `db:"queryid"`
Documentid pgtype.UUID `db:"documentid"`
Value string `db:"value"`
Cleanversion int32 `db:"cleanversion"`
Textversion int32 `db:"textversion"`
Queryversion int32 `db:"queryversion"`
}
// SetResult
//
// INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6)
func (q *Queries) SetResult(ctx context.Context, arg *SetResultParams) error {
_, err := q.db.Exec(ctx, setResult,
arg.Queryid,
arg.Documentid,
arg.Value,
arg.Cleanversion,
arg.Textversion,
arg.Queryversion,
)
return err
}
internal/database/repository/result_test.go
+215 -92
View File
@@ -44,38 +44,98 @@ func TestResults(t *testing.T) {
assert.NoError(t, err)
jobId, err := queries.CreateJob(ctx, clientId)
assert.NoError(t, err)
issynced, err := queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, issynced)
documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Jobid: jobId,
Hash: "example_hash",
})
assert.NoError(t, err)
jsonQuery, err := queries.GetQuery(ctx, jsonQueryID)
issynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, issynced)
version, err := queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Versionid: version,
Jobid: jobId,
})
assert.NoError(t, err)
cleanVersion := int32(1)
textVersion := int32(1)
issynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, issynced)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
})
assert.NoError(t, err)
issynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, issynced)
cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID)
assert.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
issynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, issynced)
textentry, err := queries.GetDocumentTextEntry(ctx, documentID)
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobId,
Queryid: jsonQueryID,
Addedversion: 1,
Name: "example_key",
})
assert.NoError(t, err)
issynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, issynced)
jsonResultValue := "example_value"
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Documentid: documentID,
Value: jsonResultValue,
Cleanversion: cleanVersion,
Textversion: textVersion,
Queryversion: jsonQuery.Activeversion,
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
issynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, issynced)
qv := int32(1)
res, err := queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{
Queryid: jsonQueryID,
Queryversion: jsonQuery.Activeversion,
Queryversion: &qv,
Documentid: documentID,
Cleanversion: cleanVersion,
Textversion: textVersion,
})
assert.NoError(t, err)
assert.EqualExportedValues(t, jsonResultValue, res.Value)
assert.NotNil(t, res.Value)
assert.Equal(t, jsonResultValue, *res.Value)
}
func TestResultValues(t *testing.T) {
@@ -147,58 +207,79 @@ func TestResultValues(t *testing.T) {
qResults, err := queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 0)
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentTwoID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 0)
result := repository.SetResultParams{
version, err := queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Versionid: version,
Jobid: jobId,
})
assert.NoError(t, err)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
})
assert.NoError(t, err)
cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID)
assert.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
textentry, err := queries.GetDocumentTextEntry(ctx, documentID)
assert.NoError(t, err)
result := repository.AddResultParams{
Queryid: contextQueryID,
Documentid: documentID,
Value: "context_value_1",
Cleanversion: 1,
Textversion: 2,
Textentryid: textentry.ID,
Queryversion: contextQuery.Activeversion,
}
err = queries.SetResult(ctx, &result)
_, err = queries.AddResult(ctx, &result)
assert.NoError(t, err)
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 1)
assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{
{
Queryid: contextQueryID,
Type: repository.QuerytypeContextFull,
Value: "context_value_1",
},
}, qResults)
assert.Equal(t, contextQueryID, qResults[0].Queryid)
assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type)
assert.Equal(t, "context_value_1", *qResults[0].Value)
assert.NotEqual(t, pgtype.UUID{}, qResults[0].ID)
assert.True(t, qResults[0].ID.Valid)
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentTwoID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 0)
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Documentid: documentID,
Value: "context_value_2",
Cleanversion: 1,
Textversion: 2,
Textentryid: textentry.ID,
Queryversion: contextQuery.Activeversion - 1,
})
assert.NoError(t, err)
@@ -206,24 +287,20 @@ func TestResultValues(t *testing.T) {
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 1)
assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{
{
Queryid: contextQueryID,
Type: repository.QuerytypeContextFull,
Value: "context_value_1",
},
}, qResults)
assert.Equal(t, contextQueryID, qResults[0].Queryid)
assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type)
assert.Equal(t, "context_value_1", *qResults[0].Value)
assert.NotEqual(t, pgtype.UUID{}, qResults[0].ID)
assert.True(t, qResults[0].ID.Valid)
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Documentid: documentID,
Value: "context_value_3",
Cleanversion: 2,
Textversion: 2,
Textentryid: textentry.ID,
Queryversion: contextQuery.Activeversion,
})
assert.NoError(t, err)
@@ -231,24 +308,20 @@ func TestResultValues(t *testing.T) {
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 1)
assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{
{
Queryid: contextQueryID,
Type: repository.QuerytypeContextFull,
Value: "context_value_3",
},
}, qResults)
assert.Equal(t, contextQueryID, qResults[0].Queryid)
assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type)
assert.Equal(t, "context_value_3", *qResults[0].Value)
assert.NotEqual(t, pgtype.UUID{}, qResults[0].ID)
assert.True(t, qResults[0].ID.Valid)
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Documentid: documentID,
Value: "json_value_1",
Cleanversion: 1,
Textversion: 2,
Textentryid: textentry.ID,
Queryversion: jsonVersion,
})
assert.NoError(t, err)
@@ -256,17 +329,15 @@ func TestResultValues(t *testing.T) {
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 1)
assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{
{
Queryid: contextQueryID,
Type: repository.QuerytypeContextFull,
Value: "context_value_3",
},
}, qResults)
assert.Equal(t, contextQueryID, qResults[0].Queryid)
assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type)
assert.Equal(t, "context_value_3", *qResults[0].Value)
assert.NotEqual(t, pgtype.UUID{}, qResults[0].ID)
assert.True(t, qResults[0].ID.Valid)
}
func TestUnsyncedNoDepsQueries(t *testing.T) {
@@ -290,13 +361,11 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
assert.NoError(t, err)
jobId, err := queries.CreateJob(ctx, clientId)
assert.NoError(t, err)
collectorId, err := queries.CreateCollector(ctx, jobId)
assert.NoError(t, err)
_, err = queries.AddLatestCollectorVersion(ctx, collectorId)
version, err := queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Collectorid: collectorId,
Versionid: 1,
Versionid: version,
Jobid: jobId,
})
assert.NoError(t, err)
documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
@@ -336,7 +405,7 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collectorId,
Jobid: jobId,
Name: "example_name",
Queryid: jsonQueryID,
Addedversion: 1,
@@ -345,19 +414,52 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
qs, err := queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
})
assert.NoError(t, err)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID)
assert.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
textentry, err := queries.GetDocumentTextEntry(ctx, documentID)
assert.NoError(t, err)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs)
assert.Len(t, qs, 0)
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Documentid: documentID,
Value: "context_value",
Cleanversion: 1,
Textversion: 2,
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
@@ -368,15 +470,12 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, qs)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs)
assert.Len(t, qs, 0)
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Documentid: documentID,
Value: "context_value",
Cleanversion: 1,
Textversion: 2,
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
@@ -386,15 +485,41 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
assert.Len(t, qs, 0)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs)
assert.Len(t, qs, 0)
err = queries.SetResult(ctx, &repository.SetResultParams{
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentTwoID,
Version: 1,
Bucket: "hi",
Key: "hello",
})
assert.NoError(t, err)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
cleantwoentry, err := queries.GetDocumentCleanEntry(ctx, documentTwoID)
assert.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleantwoentry.ID,
})
assert.NoError(t, err)
texttwoentry, err := queries.GetDocumentTextEntry(ctx, documentTwoID)
assert.NoError(t, err)
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Documentid: documentTwoID,
Value: "context_value",
Cleanversion: 1,
Textversion: 2,
Textentryid: texttwoentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
@@ -407,12 +532,10 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, qs)
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Documentid: documentTwoID,
Value: "context_value",
Cleanversion: 1,
Textversion: 2,
Textentryid: texttwoentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
internal/database/repository/text.sql.go
+19 -50
View File
@@ -12,95 +12,64 @@ import (
)
const addDocumentTextEntry = `-- name: AddDocumentTextEntry :exec
INSERT INTO documentTextExtractions (documentId, version, bucket, key) VALUES ($1, $2, $3, $4)
INSERT INTO documentTextExtractions (version, bucket, key, cleanEntryId) VALUES ($1, $2, $3, $4)
`
type AddDocumentTextEntryParams struct {
Documentid pgtype.UUID `db:"documentid"`
Version int32 `db:"version"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Cleanentryid pgtype.UUID `db:"cleanentryid"`
}
// AddDocumentTextEntry
//
// INSERT INTO documentTextExtractions (documentId, version, bucket, key) VALUES ($1, $2, $3, $4)
// INSERT INTO documentTextExtractions (version, bucket, key, cleanEntryId) VALUES ($1, $2, $3, $4)
func (q *Queries) AddDocumentTextEntry(ctx context.Context, arg *AddDocumentTextEntryParams) error {
_, err := q.db.Exec(ctx, addDocumentTextEntry,
arg.Documentid,
arg.Version,
arg.Bucket,
arg.Key,
arg.Cleanentryid,
)
return err
}
const getDocumentTextEntry = `-- name: GetDocumentTextEntry :one
SELECT dc.documentId, dc.bucket, dc.key, dc.version
FROM documentTextExtractions AS dc
JOIN documents AS d ON d.id = dc.documentId
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
ORDER BY d.id DESC LIMIT 1
SELECT id, documentId, bucket, key, version, cleanEntryId
FROM currentTextEntries
WHERE documentId = $1
`
type GetDocumentTextEntryRow struct {
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
}
// GetDocumentTextEntry
//
// SELECT dc.documentId, dc.bucket, dc.key, dc.version
// FROM documentTextExtractions AS dc
// JOIN documents AS d ON d.id = dc.documentId
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
// LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
// WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
// ORDER BY d.id DESC LIMIT 1
func (q *Queries) GetDocumentTextEntry(ctx context.Context, documentid pgtype.UUID) (*GetDocumentTextEntryRow, error) {
// SELECT id, documentId, bucket, key, version, cleanEntryId
// FROM currentTextEntries
// WHERE documentId = $1
func (q *Queries) GetDocumentTextEntry(ctx context.Context, documentid pgtype.UUID) (*Currenttextentry, error) {
row := q.db.QueryRow(ctx, getDocumentTextEntry, documentid)
var i GetDocumentTextEntryRow
var i Currenttextentry
err := row.Scan(
&i.ID,
&i.Documentid,
&i.Bucket,
&i.Key,
&i.Version,
&i.Cleanentryid,
)
return &i, err
}
const isDocumentTextExtracted = `-- name: IsDocumentTextExtracted :one
SELECT EXISTS(
SELECT 1
FROM documentTextExtractions AS dc
JOIN documents AS d ON d.id = dc.documentId
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
SELECT 1 FROM currentTextEntries WHERE documentId = $1
)
`
// IsDocumentTextExtracted
//
// SELECT EXISTS(
// SELECT 1
// FROM documentTextExtractions AS dc
// JOIN documents AS d ON d.id = dc.documentId
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
// LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
// WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
// SELECT 1 FROM currentTextEntries WHERE documentId = $1
// )
func (q *Queries) IsDocumentTextExtracted(ctx context.Context, documentid pgtype.UUID) (bool, error) {
row := q.db.QueryRow(ctx, isDocumentTextExtracted, documentid)
internal/database/repository/text_test.go
+26 -17
View File
@@ -9,6 +9,7 @@ import (
"queryorchestration/internal/test"
"testing"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
)
@@ -21,11 +22,11 @@ func TestTextExtraction(t *testing.T) {
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../.."))
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
_, textup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Cfg: cfg,
RunMigrations: true,
})
defer cleanup()
defer textup()
queries := cfg.GetDBQueries()
@@ -42,16 +43,9 @@ func TestTextExtraction(t *testing.T) {
assert.NoError(t, err)
assert.NotEmpty(t, id)
isextract, err := queries.IsDocumentTextExtracted(ctx, id)
assert.NoError(t, err)
assert.False(t, isextract)
_, err = queries.GetDocumentTextEntry(ctx, id)
assert.Error(t, err)
bucket := "example_bucket"
key := "example_key"
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: id,
Version: 1,
Bucket: bucket,
@@ -59,16 +53,31 @@ func TestTextExtraction(t *testing.T) {
})
assert.NoError(t, err)
cleanentry, err := queries.GetDocumentCleanEntry(ctx, id)
assert.NoError(t, err)
isextract, err := queries.IsDocumentTextExtracted(ctx, id)
assert.NoError(t, err)
assert.False(t, isextract)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: bucket,
Key: key,
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
isextract, err = queries.IsDocumentTextExtracted(ctx, id)
assert.NoError(t, err)
assert.True(t, isextract)
clean, err := queries.GetDocumentTextEntry(ctx, id)
text, err := queries.GetDocumentTextEntry(ctx, id)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.GetDocumentTextEntryRow{
Documentid: id,
Bucket: bucket,
Key: key,
Version: 1,
}, clean)
assert.Equal(t, id, text.Documentid)
assert.Equal(t, bucket, text.Bucket)
assert.Equal(t, key, text.Key)
assert.Equal(t, int32(1), text.Version)
assert.NotEqual(t, pgtype.UUID{}, text.ID)
assert.True(t, text.ID.Valid)
}
internal/document/text/create_test.go
+4 -3
View File
@@ -56,11 +56,12 @@ func TestCreate(t *testing.T) {
pgxmock.NewRows([]string{"isextracted"}).
AddRow(false),
)
cleanId := database.MustToDBUUID(uuid.New())
pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}).
AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key, int32(1)),
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version"}).
AddRow(cleanId, database.MustToDBUUID(id), inloc.Bucket, inloc.Key, int32(1)),
)
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(database.MustToDBUUID(id), int32(1), inloc.Bucket, inloc.Key).
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(int32(1), inloc.Bucket, inloc.Key, cleanId).
WillReturnResult(pgxmock.NewResult("", 1))
mockSQS.EXPECT().
internal/document/text/extract.go
+4 -4
View File
@@ -41,10 +41,10 @@ func (s *Service) extract(ctx context.Context, id uuid.UUID) error {
version := s.svc.Version.GetVersion()
err = s.cfg.GetDBQueries().AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Documentid: docId,
Version: version,
Bucket: outLocation.Bucket,
Key: outLocation.Key,
Version: version,
Bucket: outLocation.Bucket,
Key: outLocation.Key,
Cleanentryid: entry.ID,
})
if err != nil {
return err
internal/document/text/extract_test.go
+4 -3
View File
@@ -37,11 +37,12 @@ func TestExtract(t *testing.T) {
Key: "/i/am/here",
}
cleanId := database.MustToDBUUID(uuid.New())
pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}).
AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key, int32(1)),
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version"}).
AddRow(cleanId, database.MustToDBUUID(id), inloc.Bucket, inloc.Key, int32(1)),
)
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(database.MustToDBUUID(id), int32(1), inloc.Bucket, inloc.Key).
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(int32(1), inloc.Bucket, inloc.Key, cleanId).
WillReturnResult(pgxmock.NewResult("", 1))
err = svc.extract(ctx, id)
internal/job/collector/create.go
+11 -33
View File
@@ -18,18 +18,13 @@ type CreateParams struct {
Fields *map[string]uuid.UUID
}
func (s *Service) Create(ctx context.Context, params *CreateParams) (uuid.UUID, error) {
func (s *Service) Create(ctx context.Context, params *CreateParams) error {
dbparams, err := s.getCreateParams(ctx, params)
if err != nil {
return uuid.Nil, err
return err
}
id, err := s.submitCreate(ctx, dbparams)
if err != nil {
return uuid.Nil, err
}
return id, nil
return s.submitCreate(ctx, dbparams)
}
type dbCreateParams struct {
@@ -102,23 +97,16 @@ func (s *Service) NormalizeFieldsToDB(ctx context.Context, ofields *map[string]u
return &dbm, nil
}
func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) (uuid.UUID, error) {
var dbID pgtype.UUID
err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error {
dID, err := qtx.CreateCollector(ctx, params.JobID)
if err != nil {
return err
}
dbID = dID
version, err := qtx.AddLatestCollectorVersion(ctx, dbID)
func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) error {
return s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error {
version, err := qtx.AddLatestCollectorVersion(ctx, params.JobID)
if err != nil {
return err
}
err = qtx.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Collectorid: dbID,
Versionid: version,
Jobid: params.JobID,
Versionid: version,
})
if err != nil {
return err
@@ -126,7 +114,7 @@ func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) (uui
if params.MinCleanVersion != nil {
err = qtx.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{
Collectorid: dbID,
Jobid: params.JobID,
Versionid: *params.MinCleanVersion,
Addedversion: version,
})
@@ -137,7 +125,7 @@ func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) (uui
if params.MinTextVersion != nil {
err = qtx.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{
Collectorid: dbID,
Jobid: params.JobID,
Versionid: *params.MinTextVersion,
Addedversion: version,
})
@@ -149,7 +137,7 @@ func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) (uui
if params.Fields != nil {
for key, field := range *params.Fields {
err = qtx.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: dbID,
Jobid: params.JobID,
Name: key,
Queryid: field,
Addedversion: version,
@@ -162,14 +150,4 @@ func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) (uui
return nil
})
if err != nil {
return uuid.Nil, err
}
id, err := database.ToUUID(dbID)
if err != nil {
return uuid.Nil, err
}
return id, nil
}
internal/job/collector/create_test.go
+6 -12
View File
@@ -27,7 +27,6 @@ func TestCreate(t *testing.T) {
svc := collector.New(cfg, &collector.Services{})
id := uuid.New()
minCleanV := int32(1)
minTextV := int32(1)
create := collector.CreateParams{
@@ -44,25 +43,20 @@ func TestCreate(t *testing.T) {
AddRow(true),
)
pool.ExpectBeginTx(pgx.TxOptions{})
pool.ExpectQuery("name: CreateCollector :one").WithArgs(database.MustToDBUUID(create.JobID)).WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(id)),
)
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(create.JobID)).WillReturnRows(
pgxmock.NewRows([]string{"version"}).
AddRow(int32(1)),
)
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1)).
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(create.JobID), int32(1)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1), *create.MinCleanVersion).
pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(create.JobID), int32(1), *create.MinCleanVersion).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1), *create.MinTextVersion).
pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(create.JobID), int32(1), *create.MinTextVersion).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(id), "example_key", database.MustToDBUUID((*create.Fields)["example_key"]), int32(1)).
pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(create.JobID), "example_key", database.MustToDBUUID((*create.Fields)["example_key"]), int32(1)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
aid, err := svc.Create(ctx, &create)
err = svc.Create(ctx, &create)
assert.NoError(t, err)
assert.Equal(t, id, aid)
}
internal/job/collector/createprivate_test.go
+6 -12
View File
@@ -81,7 +81,6 @@ func TestSubmitCreate(t *testing.T) {
minCleanV := int32(2)
minTextV := int32(4)
id := uuid.New()
params := dbCreateParams{
JobID: database.MustToDBUUID(uuid.New()),
MinCleanVersion: &minCleanV,
@@ -92,27 +91,22 @@ func TestSubmitCreate(t *testing.T) {
}
pool.ExpectBeginTx(pgx.TxOptions{})
pool.ExpectQuery("name: CreateCollector :one").WithArgs(params.JobID).WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(id)),
)
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(params.JobID).WillReturnRows(
pgxmock.NewRows([]string{"version"}).
AddRow(int32(1)),
)
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1)).
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(params.JobID, int32(1)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1), *params.MinCleanVersion).
pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(params.JobID, int32(1), *params.MinCleanVersion).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1), *params.MinTextVersion).
pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(params.JobID, int32(1), *params.MinTextVersion).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(id), "example_key", (*params.Fields)["example_key"], int32(1)).
pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(params.JobID, "example_key", (*params.Fields)["example_key"], int32(1)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
aid, err := svc.submitCreate(ctx, &params)
err = svc.submitCreate(ctx, &params)
assert.NoError(t, err)
assert.Equal(t, id, aid)
}
func TestNormalizeFieldsToDB(t *testing.T) {
internal/job/collector/get_test.go
+6 -8
View File
@@ -30,7 +30,6 @@ func TestGet(t *testing.T) {
minCleanV := int32(2)
minTextV := int32(4)
ogc := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
MinCleanVersion: minCleanV,
MinTextVersion: minTextV,
@@ -39,13 +38,13 @@ func TestGet(t *testing.T) {
},
}
pool.ExpectQuery("name: GetCollector :one").WithArgs(database.MustToDBUUID(ogc.ID)).
pool.ExpectQuery("name: GetCollector :one").WithArgs(database.MustToDBUUID(ogc.JobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(ogc.ID), database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))),
pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))),
)
coll, err := svc.Get(ctx, ogc.ID)
coll, err := svc.Get(ctx, ogc.JobID)
assert.NoError(t, err)
assert.EqualExportedValues(t, ogc, *coll)
}
@@ -66,7 +65,6 @@ func TestGetByJobID(t *testing.T) {
minCleanV := int32(2)
minTextV := int32(4)
ogc := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
MinCleanVersion: minCleanV,
MinTextVersion: minTextV,
@@ -77,8 +75,8 @@ func TestGetByJobID(t *testing.T) {
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(ogc.JobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(ogc.ID), database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))),
pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))),
)
coll, err := svc.GetByJobID(ctx, ogc.JobID)
internal/job/collector/parse.go
-1
View File
@@ -26,7 +26,6 @@ func parseDBCollector(c *repository.Fullactivecollector) (*Collector, error) {
}
return &Collector{
ID: database.MustToUUID(c.ID),
JobID: database.MustToUUID(c.Jobid),
MinCleanVersion: c.Mincleanversion,
MinTextVersion: c.Mintextversion,
internal/job/collector/parse_test.go
-3
View File
@@ -18,7 +18,6 @@ func TestParseDBCollector(t *testing.T) {
minCleanV := int32(1)
minTextV := int32(2)
ogc := Collector{
ID: uuid.New(),
JobID: uuid.New(),
MinCleanVersion: minCleanV,
MinTextVersion: minTextV,
@@ -27,7 +26,6 @@ func TestParseDBCollector(t *testing.T) {
},
}
c, err = parseDBCollector(&repository.Fullactivecollector{
ID: database.MustToDBUUID(ogc.ID),
Jobid: database.MustToDBUUID(ogc.JobID),
Mincleanversion: ogc.MinCleanVersion,
Mintextversion: ogc.MinTextVersion,
@@ -39,7 +37,6 @@ func TestParseDBCollector(t *testing.T) {
ogc.MinCleanVersion = 0
ogc.MinTextVersion = 0
c, err = parseDBCollector(&repository.Fullactivecollector{
ID: database.MustToDBUUID(ogc.ID),
Jobid: database.MustToDBUUID(ogc.JobID),
Fields: []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"])),
})
internal/job/collector/service.go
-1
View File
@@ -9,7 +9,6 @@ import (
)
type Collector struct {
ID uuid.UUID
JobID uuid.UUID
MinCleanVersion int32
MinTextVersion int32
internal/job/collector/update/update.go
+7 -9
View File
@@ -183,8 +183,6 @@ func (s *Service) normalizeUpdateFieldsToDB(ctx context.Context, current map[str
func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector, params *dbUpdateParams) error {
err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error {
id := database.MustToDBUUID(current.ID)
activeName, err := validation.GetFieldName(params, params.ActiveVersion)
if err != nil {
return err
@@ -193,7 +191,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector
var latestVersion int32
if !onlyactive {
latestVersion, err = qtx.AddLatestCollectorVersion(ctx, id)
latestVersion, err = qtx.AddLatestCollectorVersion(ctx, params.JobID)
if err != nil {
return err
}
@@ -201,8 +199,8 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector
if params.ActiveVersion != nil {
err := qtx.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Collectorid: id,
Versionid: *params.ActiveVersion,
Jobid: params.JobID,
Versionid: *params.ActiveVersion,
})
if err != nil {
return err
@@ -211,7 +209,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector
if params.MinCleanVersion != nil {
err = qtx.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{
Collectorid: id,
Jobid: params.JobID,
Versionid: *params.MinCleanVersion,
Addedversion: latestVersion,
})
@@ -222,7 +220,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector
if params.MinTextVersion != nil {
err = qtx.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{
Collectorid: id,
Jobid: params.JobID,
Versionid: *params.MinTextVersion,
Addedversion: latestVersion,
})
@@ -235,7 +233,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector
removeIDs := getRemoveFields(current.Fields, params.Fields)
for _, field := range removeIDs {
err := qtx.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
Collectorid: id,
Jobid: params.JobID,
Queryid: field,
Removedversion: &latestVersion,
})
@@ -247,7 +245,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector
addIDs := getAddFields(current.Fields, params.Fields)
for key, field := range addIDs {
err := qtx.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: id,
Jobid: params.JobID,
Name: key,
Queryid: field,
Addedversion: latestVersion,
internal/job/collector/update/update_test.go
+5 -6
View File
@@ -44,7 +44,6 @@ func TestUpdate(t *testing.T) {
})
current := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
ActiveVersion: 1,
LatestVersion: 4,
@@ -59,18 +58,18 @@ func TestUpdate(t *testing.T) {
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(update.JobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(current.ID), database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")),
pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")),
)
pool.ExpectBeginTx(pgx.TxOptions{})
rv := current.LatestVersion + 1
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.ID)).WillReturnRows(
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.JobID)).WillReturnRows(
pgxmock.NewRows([]string{"version"}).
AddRow(rv),
)
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ID), int32(2)).
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(2)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.ID), rv, *update.MinCleanVersion).
pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), rv, *update.MinCleanVersion).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
internal/job/collector/update/updateprivate_test.go
+9 -11
View File
@@ -192,7 +192,6 @@ func TestSubmitUpdate(t *testing.T) {
minCleanV := int32(2)
minTextV := int32(4)
current := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
ActiveVersion: 1,
LatestVersion: 1,
@@ -219,24 +218,24 @@ func TestSubmitUpdate(t *testing.T) {
pool.ExpectBeginTx(pgx.TxOptions{})
rv := int32(2)
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.ID)).WillReturnRows(
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.JobID)).WillReturnRows(
pgxmock.NewRows([]string{"version"}).
AddRow(rv),
)
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ID), int32(2)).
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(2)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.ID), rv, *params.MinCleanVersion).
pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), rv, *params.MinCleanVersion).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(current.ID), rv, *params.MinTextVersion).
pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), rv, *params.MinTextVersion).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: RemoveCollectorQuery :exec").WithArgs(&rv, database.MustToDBUUID(current.Fields["original_key"]), database.MustToDBUUID(current.ID)).
pool.ExpectExec("name: RemoveCollectorQuery :exec").WithArgs(&rv, database.MustToDBUUID(current.Fields["original_key"]), database.MustToDBUUID(current.JobID)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.MatchExpectationsInOrder(false)
pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ID), "example_key", (*params.Fields)["example_key"], int32(2)).
pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.JobID), "example_key", (*params.Fields)["example_key"], int32(2)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ID), "second_key", (*params.Fields)["second_key"], int32(2)).
pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.JobID), "second_key", (*params.Fields)["second_key"], int32(2)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ID), "changed_key", (*params.Fields)["changed_key"], int32(2)).
pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.JobID), "changed_key", (*params.Fields)["changed_key"], int32(2)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
@@ -261,7 +260,6 @@ func TestSubmitUpdate(t *testing.T) {
}
current := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
ActiveVersion: 1,
LatestVersion: 1,
@@ -276,7 +274,7 @@ func TestSubmitUpdate(t *testing.T) {
}
pool.ExpectBeginTx(pgx.TxOptions{})
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ID), int32(2)).
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(2)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
internal/job/create.go
+1 -1
View File
@@ -16,7 +16,7 @@ func (s *Service) Create(ctx context.Context, clientID uuid.UUID) (uuid.UUID, er
id := database.MustToUUID(did)
_, err = s.svc.Collector.Create(ctx, &collector.CreateParams{
err = s.svc.Collector.Create(ctx, &collector.CreateParams{
JobID: id,
})
if err != nil {
internal/job/create_test.go
+2 -7
View File
@@ -34,22 +34,17 @@ func TestCreate(t *testing.T) {
ID: uuid.New(),
ClientID: uuid.New(),
}
collId := uuid.New()
pool.ExpectQuery("name: CreateJob :one").WithArgs(database.MustToDBUUID(job.ClientID)).WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(job.ID)),
)
pool.ExpectBeginTx(pgx.TxOptions{})
pool.ExpectQuery("name: CreateCollector :one").WithArgs(database.MustToDBUUID(job.ID)).WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(collId)),
)
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(collId)).WillReturnRows(
pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(job.ID)).WillReturnRows(
pgxmock.NewRows([]string{"version"}).
AddRow(int32(1)),
)
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(collId), int32(1)).
pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(job.ID), int32(1)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
internal/job/get.go
+22
View File
@@ -29,3 +29,25 @@ func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Job, error) {
CanSync: canSync,
}, nil
}
type JobWithStatus struct {
Job *Job
Status Status
}
func (s *Service) GetWithStatus(ctx context.Context, id uuid.UUID) (*JobWithStatus, error) {
job, err := s.Get(ctx, id)
if err != nil {
return nil, err
}
status, err := s.GetStatus(ctx, id)
if err != nil {
return nil, err
}
return &JobWithStatus{
Job: job,
Status: *status,
}, nil
}
internal/job/get_test.go
+49
View File
@@ -73,3 +73,52 @@ func TestGet(t *testing.T) {
CanSync: true,
}, ajob)
}
func TestGetWithStatus(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := job.New(cfg, &job.Services{
Collector: collector.New(cfg, &collector.Services{}),
Client: client.New(cfg),
})
j := job.Job{
ID: uuid.New(),
ClientID: uuid.New(),
CanSync: true,
}
pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows(
pgxmock.NewRows([]string{"id", "clientId", "canSync"}).
AddRow(database.MustToDBUUID(j.ID), database.MustToDBUUID(j.ClientID), j.CanSync),
)
pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "name", "canSync"}).
AddRow(database.MustToDBUUID(j.ClientID), "client_name", false),
)
pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows(
pgxmock.NewRows([]string{"issynced"}).
AddRow(true),
)
ajob, err := svc.GetWithStatus(ctx, j.ID)
assert.NoError(t, err)
jStatus := job.IN_SYNC
assert.EqualExportedValues(t, &job.JobWithStatus{
Job: &job.Job{
ID: j.ID,
ClientID: j.ClientID,
CanSync: false,
},
Status: jStatus,
}, ajob)
}
internal/job/status.go
+30
View File
@@ -0,0 +1,30 @@
package job
import (
"context"
"queryorchestration/internal/database"
"github.com/google/uuid"
)
type Status string
const (
NOT_SYNCED Status = "not_synced"
IN_SYNC Status = "in_sync"
)
func (s *Service) GetStatus(ctx context.Context, id uuid.UUID) (*Status, error) {
issynced, err := s.cfg.GetDBQueries().IsJobSynced(ctx, database.MustToDBUUID(id))
if err != nil {
return nil, err
}
status := NOT_SYNCED
switch issynced {
case true:
status = IN_SYNC
}
return &status, nil
}
internal/job/status_test.go
+88
View File
@@ -0,0 +1,88 @@
package job_test
import (
"context"
"errors"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/job"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
)
func TestGetStatus(t *testing.T) {
t.Run("is synced", func(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := job.New(cfg, &job.Services{})
jobId := uuid.New()
pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(jobId)).WillReturnRows(
pgxmock.NewRows([]string{"issynced"}).
AddRow(true),
)
issynced, err := svc.GetStatus(ctx, jobId)
assert.NoError(t, err)
assert.Equal(t, job.IN_SYNC, *issynced)
})
t.Run("not synced", func(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := job.New(cfg, &job.Services{})
jobId := uuid.New()
pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(jobId)).WillReturnRows(
pgxmock.NewRows([]string{"issynced"}).
AddRow(false),
)
issynced, err := svc.GetStatus(ctx, jobId)
assert.NoError(t, err)
assert.Equal(t, job.NOT_SYNCED, *issynced)
})
t.Run("db error", func(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := job.New(cfg, &job.Services{})
jobId := uuid.New()
errStr := "db fail"
pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(jobId)).
WillReturnError(errors.New(errStr))
_, err = svc.GetStatus(ctx, jobId)
assert.EqualError(t, err, errStr)
})
}
internal/query/result/get.go
+11 -10
View File
@@ -2,6 +2,7 @@ package result
import (
"context"
"errors"
"fmt"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
@@ -13,27 +14,27 @@ import (
)
type GetValueWithVersionParams struct {
Type resultprocessor.Type
QueryID uuid.UUID
DocumentID uuid.UUID
QueryVersion int32
MinCleanVersion int32
MinTextVersion int32
Type resultprocessor.Type
QueryID uuid.UUID
DocumentID uuid.UUID
QueryVersion int32
}
func (s *Service) GetValueWithVersion(ctx context.Context, params *GetValueWithVersionParams) (resultprocessor.Value, error) {
res, err := s.cfg.GetDBQueries().GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{
Queryid: database.MustToDBUUID(params.QueryID),
Queryversion: params.QueryVersion,
Queryversion: &params.QueryVersion,
Documentid: database.MustToDBUUID(params.DocumentID),
Cleanversion: params.MinCleanVersion,
Textversion: params.MinTextVersion,
})
if err != nil {
return nil, err
}
return getValueByType(params.Type, res.Value)
if res.Value == nil {
return nil, errors.New("no value found")
}
return getValueByType(params.Type, *res.Value)
}
func getValueByType(t resultprocessor.Type, value string) (resultprocessor.Value, error) {
internal/query/result/get_test.go
+9 -10
View File
@@ -42,23 +42,22 @@ func TestGetValueWithVersion(t *testing.T) {
svc := New(cfg, &Services{})
params := &GetValueWithVersionParams{
Type: resultprocessor.TypeJsonExtractor,
QueryID: uuid.New(),
DocumentID: uuid.New(),
QueryVersion: 1,
MinCleanVersion: 2,
MinTextVersion: 3,
Type: resultprocessor.TypeJsonExtractor,
QueryID: uuid.New(),
DocumentID: uuid.New(),
QueryVersion: 1,
}
pool.ExpectQuery("name: GetResultValueWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion, database.MustToDBUUID(params.DocumentID), params.MinCleanVersion, params.MinTextVersion).
value := "exaple_value"
pool.ExpectQuery("name: GetResultValueWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), &params.QueryVersion, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "value"}).
AddRow(pgtype.UUID{}, "example_value"),
AddRow(pgtype.UUID{}, &value),
)
val, err := svc.GetValueWithVersion(ctx, params)
assert.NoError(t, err)
v := jsonextractor.NewResult("example_value")
v := jsonextractor.NewResult(value)
assert.Equal(t, v, val)
assert.Equal(t, "example_value", v.GetStoreValue())
assert.Equal(t, value, v.GetStoreValue())
}
internal/query/result/process.go
+2 -2
View File
@@ -71,7 +71,7 @@ func (s *Service) listRequiredValues(ctx context.Context, p *Process, query *res
qResults, err := s.cfg.GetDBQueries().ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: database.MustToDBUUID(p.QueryID),
Documentid: database.MustToDBUUID(p.DocumentID),
Version: p.QueryVersion,
Version: &p.QueryVersion,
})
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return nil, err
@@ -92,7 +92,7 @@ func parseQueryRequirementValueArray(v []*repository.ListQueryRequirementValuesR
return nil, err
}
cleanValue, err := getValueByType(qType, r.Value)
cleanValue, err := getValueByType(qType, *r.Value)
if err != nil {
return nil, err
}
internal/query/result/process_test.go
+16 -13
View File
@@ -11,7 +11,6 @@ import (
"testing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
)
@@ -51,14 +50,15 @@ func TestProcess(t *testing.T) {
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)),
)
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version).
strVal := `{"examplekey":"example_value"}`
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"queryId", "type", "value"}).
AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeContextFull, "{\"examplekey\":\"example_value\"}"),
pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
AddRow(database.MustToDBUUID(uuid.New()), database.MustToDBUUID(query.ID), repository.QuerytypeContextFull, &strVal),
)
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(qcfg)),
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows(
pgxmock.NewRows([]string{"config"}).
AddRow([]byte(qcfg)),
)
val, err := svc.Process(ctx, &params)
@@ -109,16 +109,18 @@ func TestListRequiredValue(t *testing.T) {
QueryVersion: query.Version,
}
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version).
strVal := "axe_value"
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"queryId", "type", "value"}).
AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeJsonExtractor, "example_value"),
pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
AddRow(database.MustToDBUUID(uuid.New()), database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeJsonExtractor, &strVal),
)
pr, err = svc.listRequiredValues(ctx, &params, query)
assert.NoError(t, err)
assert.ElementsMatch(t, []resultprocessor.Value{
jsonextractor.NewResult("example_value"),
jsonextractor.NewResult(strVal),
}, pr)
}
@@ -138,10 +140,11 @@ func TestGetProcessor(t *testing.T) {
}
func TestParseQueryRequirementValueArray(t *testing.T) {
exval := "exampleval"
in := []*repository.ListQueryRequirementValuesRow{
{
Queryid: database.MustToDBUUID(uuid.New()),
Value: "example_value",
Value: &exval,
Type: repository.QuerytypeJsonExtractor,
},
}
@@ -149,6 +152,6 @@ func TestParseQueryRequirementValueArray(t *testing.T) {
out, err := parseQueryRequirementValueArray(in)
assert.NoError(t, err)
assert.ElementsMatch(t, []resultprocessor.Value{
jsonextractor.NewResult("example_value"),
jsonextractor.NewResult(exval),
}, out)
}
internal/query/result/processor/parse_test.go
+1 -1
View File
@@ -13,7 +13,7 @@ import (
func TestParseDBCollectorQuery(t *testing.T) {
dbResult := repository.Collectorquerydependencytree{
Collectorid: pgtype.UUID{},
Jobid: pgtype.UUID{},
Queryid: pgtype.UUID{},
Requiredids: []pgtype.UUID{},
Type: repository.QuerytypeJsonExtractor,
internal/query/result/set/set.go
+35 -14
View File
@@ -2,6 +2,8 @@ package resultset
import (
"context"
"database/sql"
"errors"
"log/slog"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
@@ -17,11 +19,6 @@ type Set struct {
func (s *Service) Set(ctx context.Context, params *Set) error {
dbid := database.MustToDBUUID(params.DocumentID)
cleanVersion, err := s.cfg.GetDBQueries().GetDocumentCleanEntry(ctx, dbid)
if err != nil {
return err
}
textVersion, err := s.cfg.GetDBQueries().GetDocumentTextEntry(ctx, dbid)
if err != nil {
return err
@@ -41,13 +38,37 @@ func (s *Service) Set(ctx context.Context, params *Set) error {
return err
}
err = s.cfg.GetDBQueries().SetResult(ctx, &repository.SetResultParams{
Queryid: database.MustToDBUUID(params.QueryID),
Documentid: database.MustToDBUUID(params.DocumentID),
Value: value.GetStoreValue(),
Cleanversion: cleanVersion.Version,
Textversion: textVersion.Version,
Queryversion: query.ActiveVersion,
err = s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
id, err := q.AddResult(ctx, &repository.AddResultParams{
Queryid: database.MustToDBUUID(params.QueryID),
Value: value.GetStoreValue(),
Queryversion: query.ActiveVersion,
Textentryid: textVersion.ID,
})
if err != nil {
return err
}
qResults, err := q.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: database.MustToDBUUID(params.QueryID),
Documentid: database.MustToDBUUID(params.DocumentID),
Version: &query.ActiveVersion,
})
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return err
}
for _, result := range qResults {
err = q.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: id,
Requiredresultid: result.ID,
})
if err != nil {
return err
}
}
return nil
})
if err != nil {
return err
@@ -60,8 +81,8 @@ func (s *Service) Set(ctx context.Context, params *Set) error {
func (s *Service) informQueryDependents(ctx context.Context, params *Set) error {
ids, err := s.cfg.GetDBQueries().ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: database.MustToDBUUID(params.DocumentID),
Requiredids: database.MustToDBUUID(params.QueryID),
Documentid: database.MustToDBUUID(params.DocumentID),
Queryid: database.MustToDBUUID(params.QueryID),
})
if err != nil {
return err
internal/query/result/set/set_test.go
+25 -15
View File
@@ -16,7 +16,6 @@ import (
"github.com/aws/aws-sdk-go-v2/service/sqs"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
@@ -65,15 +64,11 @@ func TestSet(t *testing.T) {
QueryID: query.ID,
}
pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}).
AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)),
)
textEntryId := uuid.New()
pool.ExpectQuery("name: GetDocumentTextEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}).
AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)),
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "cleanEntryId"}).
AddRow(database.MustToDBUUID(textEntryId), database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1), database.MustToDBUUID(uuid.New())),
)
pool.ExpectQuery("name: GetQuery :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows(
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
@@ -83,17 +78,32 @@ func TestSet(t *testing.T) {
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)),
)
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version).
requiredResultId := uuid.New()
strVal := "{\"examplekey\":\"example_value\"}"
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"queryId", "type", "value"}).
AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, "{\"examplekey\":\"example_value\"}"),
pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
AddRow(database.MustToDBUUID(requiredResultId), database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, &strVal),
)
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(qcfg)),
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows(
pgxmock.NewRows([]string{"config"}).
AddRow([]byte(qcfg)),
)
pool.ExpectExec("name: SetResult :exec").WithArgs(database.MustToDBUUID(query.ID), database.MustToDBUUID(params.DocumentID), pgxmock.AnyArg(), int32(1), int32(1), query.Version).
pool.ExpectBegin()
resultId := uuid.New()
pool.ExpectQuery("name: AddResult :one").WithArgs(database.MustToDBUUID(query.ID), pgxmock.AnyArg(), database.MustToDBUUID(textEntryId), query.Version).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(resultId)),
)
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
AddRow(database.MustToDBUUID(requiredResultId), database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, &strVal),
)
pool.ExpectExec("name: AddResultDependency :exec").WithArgs(database.MustToDBUUID(resultId), database.MustToDBUUID(requiredResultId)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(database.MustToDBUUID(params.QueryID), database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"queryId"}).
internal/query/test/test.go
+3 -15
View File
@@ -6,22 +6,10 @@ import (
)
func (s *Service) Test(ctx context.Context, filters result.Process) (string, error) {
doc, err := s.svc.Document.Get(ctx, filters.DocumentID)
if err != nil {
return "", err
}
col, err := s.svc.Collector.GetByJobID(ctx, doc.JobID)
if err != nil {
return "", err
}
res, err := s.svc.Result.GetValueWithVersion(ctx, &result.GetValueWithVersionParams{
QueryID: filters.QueryID,
DocumentID: filters.DocumentID,
QueryVersion: filters.QueryVersion,
MinCleanVersion: col.MinCleanVersion,
MinTextVersion: col.MinTextVersion,
QueryID: filters.QueryID,
DocumentID: filters.DocumentID,
QueryVersion: filters.QueryVersion,
})
if err == nil {
return res.GetValue(ctx)
internal/query/test/test_test.go
+8 -23
View File
@@ -43,14 +43,8 @@ func TestTest(t *testing.T) {
}),
})
coll := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
}
doc := document.Document{
ID: coll.JobID,
JobID: coll.JobID,
Hash: "example_hash",
ID: uuid.New(),
}
params := &result.Process{
QueryID: uuid.New(),
@@ -59,28 +53,19 @@ func TestTest(t *testing.T) {
}
reqID := database.MustToDBUUID(uuid.New())
pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "hash"}).
AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash),
)
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(doc.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")),
)
pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), &params.QueryVersion).WillReturnRows(
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
AddRow(database.MustToDBUUID(params.QueryID), repository.QuerytypeJsonExtractor, int32(1), params.QueryVersion+1, []byte("{\"path\":\"oldkey\"}"), []pgtype.UUID{reqID}),
)
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(params.QueryID), params.QueryVersion).
strVal := "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}"
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.QueryID), &params.QueryVersion, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"queryId", "type", "value"}).
AddRow(reqID, repository.QuerytypeContextFull, "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}"),
pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
AddRow(database.MustToDBUUID(uuid.New()), reqID, repository.QuerytypeContextFull, &strVal),
)
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion).WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte("{\"path\":\"oldkey\"}")),
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID)).WillReturnRows(
pgxmock.NewRows([]string{"config"}).
AddRow([]byte("{\"path\":\"oldkey\"}")),
)
result, err := svc.Test(ctx, *params)
internal/query/types/jsonExtractor/process_test.go
+23 -24
View File
@@ -12,7 +12,6 @@ import (
"testing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
)
@@ -44,10 +43,10 @@ func TestJSONProcess(t *testing.T) {
config := "{\"path\":\"key\"}"
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(config)),
pgxmock.NewRows([]string{"config"}).
AddRow([]byte(config)),
)
value, err := extractor.Process(ctx, query, values)
@@ -59,10 +58,10 @@ func TestJSONProcess(t *testing.T) {
values = []resultprocessor.Value{
contextfull.NewResult(jsonString),
}
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(config)),
pgxmock.NewRows([]string{"config"}).
AddRow([]byte(config)),
)
value, err = extractor.Process(ctx, query, values)
@@ -74,10 +73,10 @@ func TestJSONProcess(t *testing.T) {
values = []resultprocessor.Value{
contextfull.NewResult(jsonString),
}
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(config)),
pgxmock.NewRows([]string{"config"}).
AddRow([]byte(config)),
)
value, err = extractor.Process(ctx, query, values)
@@ -112,10 +111,10 @@ func TestJSONProcessJSON(t *testing.T) {
config := "{\"path\":\"invalid_key\"}"
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(config)),
pgxmock.NewRows([]string{"config"}).
AddRow([]byte(config)),
)
value, err := extractor.Process(ctx, query, values)
@@ -124,10 +123,10 @@ func TestJSONProcessJSON(t *testing.T) {
config = ""
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(config)),
pgxmock.NewRows([]string{"config"}).
AddRow([]byte(config)),
)
value, err = extractor.Process(ctx, query, values)
@@ -136,10 +135,10 @@ func TestJSONProcessJSON(t *testing.T) {
config = "{\"path\":\"\"}"
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(config)),
pgxmock.NewRows([]string{"config"}).
AddRow([]byte(config)),
)
value, err = extractor.Process(ctx, query, values)
@@ -148,19 +147,19 @@ func TestJSONProcessJSON(t *testing.T) {
config = "{\"path\":}"
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(config)),
pgxmock.NewRows([]string{"config"}).
AddRow([]byte(config)),
)
value, err = extractor.Process(ctx, query, values)
assert.EqualError(t, err, "invalid character '}' looking for beginning of value")
assert.Empty(t, value)
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}),
pgxmock.NewRows([]string{"config"}),
)
value, err = extractor.Process(ctx, query, values)
internal/query/types/jsonExtractor/service.go
+2 -6
View File
@@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
resultprocessor "queryorchestration/internal/query/result/processor"
"queryorchestration/internal/serviceconfig"
@@ -34,16 +33,13 @@ func (e *Extractor) Process(ctx context.Context, query *resultprocessor.Query, v
return "", err
}
byteConfig, err := e.cfg.GetDBQueries().GetQueryConfig(ctx, &repository.GetQueryConfigParams{
Queryid: database.MustToDBUUID(query.ID),
Addedversion: query.Version,
})
byteConfig, err := e.cfg.GetDBQueries().GetActiveQueryConfig(ctx, database.MustToDBUUID(query.ID))
if err != nil {
return "", err
}
var config Config
err = json.Unmarshal(byteConfig.Config, &config)
err = json.Unmarshal(byteConfig, &config)
if err != nil {
return "", err
}
internal/query/update/update.go
+2 -2
View File
@@ -66,8 +66,8 @@ func (s *Service) normalizeUpdateRequiredQueryIDs(ctx context.Context, current *
}
createsloop, err := s.cfg.GetDBQueries().IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: database.MustToDBUUID(current.ID),
ID: database.MustToDBUUIDArray(*entity.GetRequiredQueryIDs()),
Queryid: database.MustToDBUUID(current.ID),
Requiredqueryids: database.MustToDBUUIDArray(*entity.GetRequiredQueryIDs()),
})
if err != nil {
return err
pkg/queryService/api.gen.go
+12
View File
@@ -36,6 +36,12 @@ const (
RightClosedInterval FieldFilterCondition = "right_closed_interval"
)
// Defines values for JobStatus.
const (
INSYNC JobStatus = "IN_SYNC"
NOTSYNCED JobStatus = "NOT_SYNCED"
)
// Defines values for QueryType.
const (
CONTEXTFULL QueryType = "CONTEXT_FULL"
@@ -121,6 +127,9 @@ type Job struct {
// Id The job id
Id openapi_types.UUID `json:"id"`
// Status Specifies the status of a job.
Status JobStatus `json:"status"`
}
// JobClient defines model for JobClient.
@@ -186,6 +195,9 @@ type JobCreate struct {
ClientId openapi_types.UUID `json:"client_id"`
}
// JobStatus Specifies the status of a job.
type JobStatus string
// JobUpdate defines model for JobUpdate.
type JobUpdate struct {
// CanSync Specifies whether the job is actively syncing
serviceAPIs/queryService.yaml
+76 -67
View File
@@ -32,15 +32,15 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/ClientCreate'
$ref: "#/components/schemas/ClientCreate"
responses:
'201':
"201":
description: Client created successfully.
content:
application/json:
schema:
$ref: '#/components/schemas/IdMessage'
'400':
$ref: "#/components/schemas/IdMessage"
"400":
description: Invalid request body.
/client/{id}:
@@ -59,15 +59,15 @@ paths:
summary: Get a client by ID
description: Retrieves a specific client by its ID.
responses:
'200':
"200":
description: Client details.
content:
application/json:
schema:
$ref: '#/components/schemas/JobClient'
'400':
$ref: "#/components/schemas/JobClient"
"400":
description: Invalid request parameters.
'404':
"404":
description: Client not found.
patch:
operationId: updateClient
@@ -80,13 +80,13 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/ClientUpdate'
$ref: "#/components/schemas/ClientUpdate"
responses:
'200':
"200":
description: Client updated successfully.
'400':
"400":
description: Invalid request body.
'404':
"404":
description: Client not found
/query:
@@ -97,15 +97,15 @@ paths:
summary: List queries
description: Retrieves a list of queries based on filter criteria.
responses:
'200':
"200":
description: A list of queries.
content:
application/json:
schema:
$ref: '#/components/schemas/ListQueries'
'400':
$ref: "#/components/schemas/ListQueries"
"400":
description: Invalid request parameters.
'404':
"404":
description: Query not found.
post:
operationId: createQuery
@@ -118,15 +118,15 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/QueryCreate'
$ref: "#/components/schemas/QueryCreate"
responses:
'201':
"201":
description: Query created successfully.
content:
application/json:
schema:
$ref: '#/components/schemas/IdMessage'
'400':
$ref: "#/components/schemas/IdMessage"
"400":
description: Invalid request body.
/query/{id}:
@@ -145,15 +145,15 @@ paths:
summary: Get a query by ID
description: Retrieves a specific query by its ID.
responses:
'200':
"200":
description: Query details.
content:
application/json:
schema:
$ref: '#/components/schemas/Query'
'400':
$ref: "#/components/schemas/Query"
"400":
description: Invalid request parameters.
'404':
"404":
description: Query not found.
patch:
operationId: updateQuery
@@ -166,13 +166,13 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/QueryUpdate'
$ref: "#/components/schemas/QueryUpdate"
responses:
'200':
"200":
description: Query updated successfully.
'400':
"400":
description: Invalid request body.
'404':
"404":
description: Query not found.
/query/{id}/test:
@@ -195,15 +195,15 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/QueryTestRequest'
$ref: "#/components/schemas/QueryTestRequest"
responses:
'200':
"200":
description: Test result.
content:
application/json:
schema:
$ref: '#/components/schemas/QueryTestResponse'
'400':
$ref: "#/components/schemas/QueryTestResponse"
"400":
description: Invalid request body.
/job:
@@ -218,15 +218,15 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/JobCreate'
$ref: "#/components/schemas/JobCreate"
responses:
'201':
"201":
description: Job created successfully.
content:
application/json:
schema:
$ref: '#/components/schemas/IdMessage'
'400':
$ref: "#/components/schemas/IdMessage"
"400":
description: Invalid request body.
/job/{id}:
@@ -245,15 +245,15 @@ paths:
summary: Get a job by ID
description: Retrieves a specific job by its ID.
responses:
'200':
"200":
description: Job details.
content:
application/json:
schema:
$ref: '#/components/schemas/Job'
'400':
$ref: "#/components/schemas/Job"
"400":
description: Invalid request parameters.
'404':
"404":
description: Job not found.
patch:
operationId: updateJob
@@ -266,13 +266,13 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/JobUpdate'
$ref: "#/components/schemas/JobUpdate"
responses:
'204':
"204":
description: Job updated successfully.
'400':
"400":
description: Invalid request body.
'404':
"404":
description: Job not found.
/job/{id}/collector:
@@ -291,13 +291,13 @@ paths:
summary: Get a job collector by ID
description: Retrieves a specific job collector by its ID.
responses:
'200':
"200":
description: Job collector details.
content:
application/json:
schema:
$ref: '#/components/schemas/JobCollector'
'404':
$ref: "#/components/schemas/JobCollector"
"404":
description: Job collector not found.
patch:
operationId: updateJobCollectorByJobId
@@ -310,13 +310,13 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/JobCollectorUpdate'
$ref: "#/components/schemas/JobCollectorUpdate"
responses:
'204':
"204":
description: Job collector updated successfully.
'400':
"400":
description: Invalid request body.
'404':
"404":
description: Job collector not found.
/job/export:
@@ -331,17 +331,17 @@ paths:
content:
application/json:
schema:
$ref: '#/components/schemas/ExportTrigger'
$ref: "#/components/schemas/ExportTrigger"
responses:
'201':
"201":
description: Export triggered successfully.
content:
application/json:
schema:
$ref: '#/components/schemas/IdMessage'
'400':
$ref: "#/components/schemas/IdMessage"
"400":
description: Invalid request body.
'404':
"404":
description: Job not found.
/job/{id}/export:
@@ -360,16 +360,15 @@ paths:
summary: Check export state.
description: Checks the current state of an export.
responses:
'200':
"200":
description: Export has been completed.
content:
application/json:
schema:
$ref: '#/components/schemas/ExportDetails'
'404':
$ref: "#/components/schemas/ExportDetails"
"404":
description: Export job not found.
components:
schemas:
QueryType:
@@ -387,7 +386,7 @@ components:
format: uuid
description: Unique identifier for the query.
type:
$ref: '#/components/schemas/QueryType'
$ref: "#/components/schemas/QueryType"
active_version:
type: integer
format: int32
@@ -417,7 +416,7 @@ components:
queries:
type: array
items:
$ref: '#/components/schemas/Query'
$ref: "#/components/schemas/Query"
description: List of queries.
required:
- queries
@@ -426,7 +425,7 @@ components:
type: object
properties:
type:
$ref: '#/components/schemas/QueryType'
$ref: "#/components/schemas/QueryType"
config:
type: string
description: Configuration for the new query.
@@ -498,6 +497,13 @@ components:
- name
- can_sync
JobStatus:
type: string
enum:
- IN_SYNC
- NOT_SYNCED
description: Specifies the status of a job.
Job:
type: object
properties:
@@ -512,10 +518,13 @@ components:
can_sync:
type: boolean
description: Specifies whether the job is actively syncing
status:
$ref: "#/components/schemas/JobStatus"
required:
- id
- client_id
- can_sync
- status
JobCreate:
type: object
@@ -590,7 +599,7 @@ components:
type: array
description: The fields in the job collector.
items:
$ref: '#/components/schemas/JobCollectorField'
$ref: "#/components/schemas/JobCollectorField"
description: JobCollector model.
required:
- job_id
@@ -619,7 +628,7 @@ components:
type: array
description: The fields in the job collector.
items:
$ref: '#/components/schemas/JobCollectorField'
$ref: "#/components/schemas/JobCollectorField"
description: Payload for updating a JobCollector.
JobCollectorField:
@@ -658,7 +667,7 @@ components:
type: array
description: Filter the scope based on field output values.
items:
$ref: '#/components/schemas/FieldFilter'
$ref: "#/components/schemas/FieldFilter"
description: Payload for triggering an export.
required:
- job_id
@@ -711,7 +720,7 @@ components:
format: uuid
description: The job id relative to the export.
status:
$ref: '#/components/schemas/ExportStatus'
$ref: "#/components/schemas/ExportStatus"
output_location:
type: string
description: The location in which the export zip file will be found.
test/queryService/job_test.go
+2
View File
@@ -42,6 +42,7 @@ func TestJob(t *testing.T) {
assert.Equal(t, id, jobRes.JSON200.Id)
assert.Equal(t, clientRes.JSON201.Id, jobRes.JSON200.ClientId)
assert.False(t, jobRes.JSON200.CanSync)
assert.Equal(t, queryservice.INSYNC, jobRes.JSON200.Status)
updateCanSync := !jobRes.JSON200.CanSync
updateRes, err := client.UpdateJobWithResponse(ctx, id, queryservice.JobUpdate{
@@ -55,4 +56,5 @@ func TestJob(t *testing.T) {
assert.Equal(t, id, jobRes.JSON200.Id)
assert.Equal(t, clientRes.JSON201.Id, jobRes.JSON200.ClientId)
assert.False(t, jobRes.JSON200.CanSync)
assert.Equal(t, queryservice.INSYNC, jobRes.JSON200.Status)
}