diff --git a/.dockerignore b/.dockerignore index 8ebc1df2..5b93570b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -32,6 +32,8 @@ LICENSE README.md +**/*_test.go +cmd/*_test/ .bitbucket/ .devbox/ build/ @@ -55,4 +57,4 @@ sqlc.yml vaccum.conf.yaml .*.yml **/*.cast -**/*.http \ No newline at end of file +**/*.http diff --git a/api/docTextRunner/runner_test.go b/api/docTextRunner/runner_test.go index caf49d1b..5274a923 100644 --- a/api/docTextRunner/runner_test.go +++ b/api/docTextRunner/runner_test.go @@ -70,11 +70,12 @@ func TestDocCleanRunner(t *testing.T) { pgxmock.NewRows([]string{"isextracted"}). AddRow(false), ) + cleanId := database.MustToDBUUID(uuid.New()) pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(doc.ID)).WillReturnRows( - pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). - AddRow(database.MustToDBUUID(doc.ID), inloc.Bucket, inloc.Key, int32(1)), + pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version"}). + AddRow(cleanId, database.MustToDBUUID(doc.ID), inloc.Bucket, inloc.Key, int32(1)), ) - pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(database.MustToDBUUID(doc.ID), int32(1), inloc.Bucket, inloc.Key). + pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(int32(1), inloc.Bucket, inloc.Key, cleanId). WillReturnResult(pgxmock.NewResult("", 1)) mockSQS.EXPECT(). diff --git a/api/queryRunner/runner_test.go b/api/queryRunner/runner_test.go index 9b45761b..56c379b2 100644 --- a/api/queryRunner/runner_test.go +++ b/api/queryRunner/runner_test.go @@ -21,7 +21,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/go-playground/validator/v10" "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -81,15 +80,12 @@ func TestQueryRunner(t *testing.T) { DocumentID: doc.DocumentID, } - pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)). - WillReturnRows( - pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). - AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)), - ) + cleanEntryId := uuid.New() + textEntryId := uuid.New() pool.ExpectQuery("name: GetDocumentTextEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)). WillReturnRows( - pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). - AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)), + pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "cleanEntryId"}). + AddRow(database.MustToDBUUID(textEntryId), database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1), database.MustToDBUUID(cleanEntryId)), ) pool.ExpectQuery("name: GetQuery :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows( pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). @@ -99,17 +95,32 @@ func TestQueryRunner(t *testing.T) { pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)), ) - pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version). + requiredResultId := uuid.New() + strVal := "{\"examplekey\":\"example_value\"}" + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)). WillReturnRows( - pgxmock.NewRows([]string{"queryId", "type", "value"}). - AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, "{\"examplekey\":\"example_value\"}"), + pgxmock.NewRows([]string{"id", "queryId", "type", "value"}). + AddRow(database.MustToDBUUID(requiredResultId), database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, &strVal), ) - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(qcfg)), + pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows( + pgxmock.NewRows([]string{"config"}). + AddRow([]byte(qcfg)), ) - pool.ExpectExec("name: SetResult :exec").WithArgs(database.MustToDBUUID(query.ID), database.MustToDBUUID(params.DocumentID), pgxmock.AnyArg(), int32(1), int32(1), query.Version). + pool.ExpectBegin() + resultId := uuid.New() + pool.ExpectQuery("name: AddResult :one").WithArgs(database.MustToDBUUID(query.ID), pgxmock.AnyArg(), database.MustToDBUUID(textEntryId), query.Version). + WillReturnRows( + pgxmock.NewRows([]string{"id"}). + AddRow(database.MustToDBUUID(resultId)), + ) + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)). + WillReturnRows( + pgxmock.NewRows([]string{"id", "queryId", "type", "value"}). + AddRow(database.MustToDBUUID(requiredResultId), database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, &strVal), + ) + pool.ExpectExec("name: AddResultDependency :exec").WithArgs(database.MustToDBUUID(resultId), database.MustToDBUUID(requiredResultId)). WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectCommit() pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(database.MustToDBUUID(query.ID), database.MustToDBUUID(doc.DocumentID)). WillReturnRows( pgxmock.NewRows([]string{"queryId"}). diff --git a/api/queryService/api.gen.go b/api/queryService/api.gen.go index c526a144..1a0ad992 100644 --- a/api/queryService/api.gen.go +++ b/api/queryService/api.gen.go @@ -38,6 +38,12 @@ const ( RightClosedInterval FieldFilterCondition = "right_closed_interval" ) +// Defines values for JobStatus. +const ( + INSYNC JobStatus = "IN_SYNC" + NOTSYNCED JobStatus = "NOT_SYNCED" +) + // Defines values for QueryType. const ( CONTEXTFULL QueryType = "CONTEXT_FULL" @@ -123,6 +129,9 @@ type Job struct { // Id The job id Id openapi_types.UUID `json:"id"` + + // Status Specifies the status of a job. + Status JobStatus `json:"status"` } // JobClient defines model for JobClient. @@ -188,6 +197,9 @@ type JobCreate struct { ClientId openapi_types.UUID `json:"client_id"` } +// JobStatus Specifies the status of a job. +type JobStatus string + // JobUpdate defines model for JobUpdate. type JobUpdate struct { // CanSync Specifies whether the job is actively syncing @@ -598,47 +610,48 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+RbX2/bthb/KoTufTTsdN2T37o0HRz0rr1tejFgCAyKOraZ0aJCUk69wN/9gjzUP0u0", - "5CTaMuypTUSeQ/7O7/whD/MYMbnNZAqp0dH8MdJsA1vq/nspOKTmUgE1YH/OlMxAGQ7ua0q37rcJaKZ4", - "ZrhMo3l0swHC3DziBkwis88gmkfaKJ6uo8NhEim4z7mCJJr/hlJuy1EyvgNmosPEK/+WJZ3KGU2Xep+y", - "9gIWK2KqNXBNqBDygadrQpnhOyB2mq7WFUspgKZW5dN31Fr91fdMKvMeDOVCt2V+pnshaUJWUhFwQ4lR", - "fL0GRRToTKYaptHkaM93Ml7ypHuBdzImPCEKBHWbNNKhgLKtqJVUW2qieZTnPGnvYRLJ3GS5WQrJKMrt", - "UlN8JTwlDxvONjUt5A+ekRUXQB64ECQGspJ5mljl8J1uM+H0vZ3PZo9xzn4Hc5g9Iq48Ocwe72Ts/jV8", - "C9rQbXZYPqJgnhymf/Csa9HaUJM7cP6tYBXNo3/NKjbPPJVnaIyvOPaYgB7VUtZt0JpfS2VtYDKpNY9F", - "iYW1hxUI2u0/zbdWl12bAANWHU+XmZJrBdqScUW5gKSmvNojKr9BepymkueQI3taM36TRysOIlmuuDCg", - "OrbzwX1whtVMZkBiqiEhMiVuIkGikB0VOe6OG9j22uCDnYuio8phqFJ0b3/m6Rq0XcBT1lVOJhlVdAt2", - "fnvbkCbLIph0MJtqQ+xnIleVwGmAdMoERXFNVlwNFNYVOga4ucQYd5+D2tuAkQujrcvHBf8gGeDz3Y7Q", - "xf+68QJ2cZwjTIp8m7agZzJNeBFVBrLkspxzmHjKhuOz/VKggizlqYUnbESkb7c0/EZyDatcFKEUedng", - "e0tok9VH+Nb2MKkhUi6lB/jLOoYnAhBuf1UapdTUiEQCtF6aDbX61y7Bq+JHJqSGZMlTA2pHRTSJZAZp", - "/WcBK7NsD1N8ven6PU+ZyBNweQD/1xXmFsl/QGu67kj2Xd7wLeX3ORCe2Pyx4qAwl6aGm/353A/w/lrG", - "55QeXzNgdi2aPGzAbHywcj6rffUh9q7+sGvoqkAwHwbdvyhqklJwDEKma+v8Q3L86bDyFNTqS55U0ATA", - "xJruT6rmekEcgtjLFLlOduH5fRhJIYAZ2RFp61/JViYg2lkOYVnuQOlgrPDQ+TFF3GSF6Ib78NS8/aHa", - "oHXrNaZwF2gCERS/2SBcELUhfVDFUN+ti4NddcOpZLl4X2yNai0ZpwYSu5ZB5bCwtZs5jSOOeS6OW57y", - "bb5dMktcUKdV+sGlTlf1bYAkkuVby0cv5UzdBr6bpym2M23ZpQfqDFXfnk1hOAKLbVlqcuwCfX6G3ArT", - "mFQOVu66xeihR2NfPKNgO6yzOHGFXZDXWPa5JEANHrYymeUWB7c4LAjPT4I+QJXK+4CrzubhE0lux2Bt", - "WJ/6j49b/zin72RS4GZp1CroiPSVrgDdz7+Bem4Z2IXWR67Nf3NQXnlzKffVh+ZK7CzrJn7AYApbTfve", - "A02htgs4lNBa6dO93EWlgQRnMl3xdVv6pft9rvAKqyBzKXlQudx97uhe4cvXFuegUJhq2UuPYqTPLIv3", - "Tab07uc4uOHPAyh2Ywd2FstORCuVt6ALUi8YWs6iRgoPJ+jxdwXYzQ4idwPafAF3d9KGr8g4nbG5qrfL", - "xGQkcTSma8pTPewiGuuPoHP8r8MjrJ5cA9oNtC04nlKM1nd3vI4evPDOvg2Yu9tpb+KLK9GO9gAI0Ol8", - "hQLDq/HMCGUkl733GbQCSnExdP310y/Lq19vvry7vPn0JZpEl59+ubn69Wb54dvHj50XN05vKEv2BXyc", - "lxwF/WfG+UIoOzPe9zt0IVmM4tjtxH9wV+Mr2V7Ju8+L0s2a+3P2IJ8U24A2fu8a1I4zrAAMN64V0zXu", - "3edFNIlKa0VvphfTC9cdyiClGY/m0dvpxfStLd+p2bhNzlh1qyMxaBzFVheLNaEunvrq7YEbbB5lSu54", - "AglJsFk2xQtHXNAiKef7uyO0Emjzk0z2PqIbr55mmeDYoJrdaaQbhsO+YNnodB6aPmdUDu4X6ONuzz9c", - "vHkx3dXFp1N8hB2ixdzKEqJzxkDrVS7Efmrt8uPFRUcgTndUuG6gQ4rEMrGjD5NI59sttYWZB7VhE8sN", - "utY2CqDar8ia6NZO9WaePfLkYFWuwXTFNaM47JyxNQYdVlg83hNuNFm8b1v4ZzA18zZwvngxnKv7xzDO", - "JQeHQltvOLk5P3bw398USlN0RJum+BkMoTWUFu9PWMI6XqEymv92+t7LyzSSKG8YF57sQOu+xaXkHGuu", - "JuUnNVj7jlS3LhywTShgamxHcpeZGxHAsi/o+Dj5T3B8n70GOf5F0MC5Tw7PctKBHDqiEG6gZFGPJ9/5", - "tsaAaG1PrWeH6msZj2Su6tLgNQXpaxmPHaHvHKSFUa9l3LLoDDu/YcMuUm64M23tzUampF1u25L+vQE+", - "PhjJms2XDa/JoleNNzkj+bSlTTApeFSqdxw18/vnKC0GnJ+a3eXZybxc+PJ4STnkUaPk4pOYYyL2mDSy", - "cMPhBqRgKwMB/auTbRnAB2TaUeP2OTk2YLgxE+xJYpTpdUAYtk44Y/Um7nnuWE7td8yyzfDT/lrGi2Ts", - "6rncVCgHlktv+m4A7mr4AI9swHLsm+XKnuCk5bm52el5BW5bbfkMB+7kxDiF2FEf8DmeXW11bB8fQLq6", - "t1fjexnXCABVMdbp/ZcbYL9jIcZypWxp716L2oNb4+Vm08rVO1QY09ebj5fD5dGGahIDpKR81xrG/qp6", - "FxsE3qFSVKYOj+mJquesk3CF6AiebS1/XzS8eqO9aDbl6s9r3QNXprgBxWnb/PUm4Ijmr6vpMP674x28", - "bIWG15JBjriWyn0JQkEON6vBjQGHW7y0Pft4i73NceJqvXn1mg5EaJWRD7n3HtiAUUs3e8JBB019uqKq", - "7DqSY/m2egjdUQ48Pe6EBVaJTqOwavvU8HhbNln+6kKq5uMDCqjRXfuZN41ozTFrpB6+lLXRWb46M0U3", - "+dUwqDM9XH0HlmN+cG1rlbs+Mw1miubfmxzdoIE2o/Op3q0fTqqX1u+73x2R7cbB6B8kPidROEH9tLNz", - "QO26CfZZySRnZUMUbE2fKxHNo40xmZ7PZjTjU//HalMmt7Pdm8iyxWs7lvepsLfGP7yDhBjpr+B1xdbm", - "FfxhMkzMnYxrMmpXDWcIqE4vTVGt08tQmVXh5YU18B8qBYvxmpRmaX+4Pfw/AAD///6KTOeVOgAA", + "H4sIAAAAAAAC/+RbW3PbNhb+KxjuPmokt+mT3lLb6ciTjbOxstOdTkYDgkcSXJCgAVCO6tF/38GFNxEQ", + "KdtsvdOnxCZwDvCd71yAAz9FhKc5zyBTMpo/RZJsIcXmv5eMQqYuBWAF+udc8ByEomC+Zjg1v01AEkFz", + "RXkWzaPlFhAx85AZMInUPodoHkklaLaJDodJJOChoAKSaP6blfKtGsXjeyAqOkyc8q954lVOcLaS+4x0", + "F7BYI1WvgUqEGeOPNNsgTBTdAdLTZL2umHMGONMqn7+jzuqvv+dcqCtQmDLZlfkZ7xnHCVpzgcAMRUrQ", + "zQYEEiBznkmYRpOjPd/zeEUT/wLveYxoggQwbDapuEHBytai1lykWEXzqCho0t3DJOKFygu1YpxgK9en", + "pvyKaIYet5RsG1rQHzRHa8oAPVLGUAxozYss0crhO05zZvS9m89mT3FBfgd1mD1ZXGlymD3d89j8q2gK", + "UuE0P6yerGCaHKZ/0Ny3aKmwKgw4/xSwjubRP2Y1m2eOyjNrjDs79piADtVK1regNe8qZV1gci4ljVmF", + "hbaHFgjS7D8rUq1Lr42BAq2OZqtc8I0Aqcm4xpRB0lBe79EqX1p6nKaS45Ahe9YwfptHawosWa0pUyA8", + "2/lgPhjDSsJzQDGWkCCeITMRWaKgHWaF3R1VkPba4IOea0VHtcNgIfBe/0yzDUi9gOesq5qMcixwCnp+", + "d9uQJasymHiYjaVC+jPi61rgNEA6oYKiqERrKgYK84WOAW7ObYx7KEDsdcAomJLa5eOSf5AM8Hm/I/j4", + "3zRewC6Gc4hwVqRZB3rCs4SWUWUgSy6rOYeJo2w4PusvJSqWpTTT8ISNaOnrl2a/oULCumBlKLW8bPG9", + "I7TN6iN8G3uYNBCpltID/GUTwxMByG5/XRml0tSKRAykXKkt1vo3JsGL8kfCuIRkRTMFYodZNIl4Dlnz", + "ZwZrteoOE3Sz9f2eZoQVCZg8YP/nC3OL5F8gJd54kr3PG75m9KEARBOdP9YUhM2lmaJqfz73A7y/4fE5", + "pcddDkSvRaLHLaitC1bGZ6WrPtje1B96Db4KxObDoPuXRU1SCY6B8WyjnX9Ijj8dVoZIGJZwb3gcyLZG", + "ar3LSY3myQR8w2NbD/5JlWCvAYZg9ToFspFdRo1yryGMOGNAFPdE6eZXlPIEWDdDWlhWOxAyGGccdG5M", + "GXNJKbrlejRT736sN6hDwsamfxOkAtHXftMBvCR5S/qgaqO5WxNDfTXHqUS7uCq3hqXkhGIFiV7LoFKa", + "6bpPncbRjnkpjinNaFqkK6KJC+K0Sje40mkqxi2ghJMi1Xx0Us7UreC7ep5iPVOXbHKgzlDl7tgUhiOw", + "2I6lJscu0OdnllthGqPawapddxg99FjtCm8rWA/zFjamKAzy2paMJoFgZQ9qOc8LjYNZnC0mz0+gLkBV", + "yvuAq8/14dNMocfYurI59W8ft/52Tu9lUuBWatQK6oj0ta4A3UM3BnWhaE6zZpSmJC5TTFmmLz6t7v77", + "6TKaRJ9ul+a/11fe6vmGx+dflb20XvWZ5iOV6t8FCKe8vZSH+kN7JXqWBsANGOwvWtO+9+RVqvVZyUro", + "rPT5IcWEwIHeRHi2ppuu9Evz+0LYu7bScyrJg+p6/wHJv8LXL2TOQaE01aqXHuVIl8YWV22m9O7nOJLa", + "nwdQbKkHeitzI6JTN3SgC1IvGMfOokYGjyfo8f8KsJkdRG4JUn0Bc8nTha9Mb95EUBf3VRZUHBka4w2m", + "mRx2Y26LnaBz/MfjEVpPIcHaDaSubp5T+TZ3d7yOHrxsc6ELmLmE6m7ii6kHj/YAFqDTydEKDK/GMeNU", + "YtQzOwGlTI03d7efVte/Lr+8v1zefokm0eXtp+X1r8vVh68fP3pzpNEbypJ9Ad/OS46C/gvjfCmUnBnv", + "+x26lMxGcexu4j+YO/w1767k/edF5Wbt/Rl7oFtBtiCV27sEsaPEVgCKKtMz8o17/3kRTaLKWtEP04vp", + "hWlj5ZDhnEbz6N30YvpOnxWw2ppNzkh9hcRt0DiKrSYWS4RNPHWl4iNVtsuVC76jCSQosV29qb0ZtQta", + "JNV8d1FlrQRS/cyTvYvoyqnHec6o7aTN7qWlmw2HfcGy1ZI9tH1OiQLML6yPmz3/ePHDq+mub2iN4iPs", + "LFrErCxBsiAEpFwXjO2n2i4/XVx4AnG2w8y0LQ1SKOaJHn2YRLJIU6wLMwdqyyaaG3gjdRSwau8sa6Jv", + "eqoz8+yJJgetcgPKF9eUoLAzxpY26JDS4vEeUSXR4qpr4V9ANczbwvni1XCuLzvDOFccHAptszNm5vzk", + "4b+7luSqbN22TfELKIQbKC2uTlhCO16pMpr/dvqSzclUHAlnGBOe9EDtvuUN6NzWXG3KTxqw9p3fvplw", + "QLahgClt35SazNyKAJp9Qce3k/8Ex3fZa5DjXwQNXLjk8CInHcihIwrZDVQs6vHke9d/GRCt9an17FB9", + "w+ORzFXfULylIH3D47Ej9L2BtDTqDY87Fp3ZFnXYsIuMKmpM23hckguul9u1pHsYYV9JjGTN9hOMt2TR", + "69bjoZF8WtMmmBQcKvWDk4b53buZDgPOT83mpu5kXi59ebykHPKoUXLxScxtInaYtLJwy+EGpGAtwwL6", + "VyfbKoAPyLSjxu1zcmzAcGMm2JPEqNLrgDCsnXBGmh3j89yxmtrvmFVP4+f9DY8XydjVc7WpUA6slt72", + "3QDc9fABHtmC5dg3q5U9w0mrc3O7rfQG3Lbe8hkO7OXEOIXYUdPxJZ5db3VsHx9Auqa31+N7GdcKAHUx", + "5vX+yy2Q320hRgohdGlvnrWablXziWnbyvWDWRjT19uvrMPl0RZLFANkqHqAG8b+un7AGwTeoFJWpgaP", + "6Ymq56yTcI3oCJ6tLf9QNrx6oz1rN+Wa74DNS1wiqAJBcdf8zSbgiOZvqvEY//3xDl63QrPXkkGOmJbK", + "QwVCSQ4zq8WNAYdbe2l79vHW9jbHiavN5tVbOhBZq4x8yH1wwAaMWrnZMw461tSnK6rariM5lmurh9Ad", + "5cDT4062wKrQaRVWXZ8aHm+rJstfXUg1fHxAATW6a7/wptFac8waqYcvVW10lq/OVNlNfjMM8qaH6+9A", + "CpsfTNtaFJl9PxTKFO0/jDm6QQOpRudTs1s/nFSvrd91vz2RbWlgdK8fX5IojKB+2uk5IHZ+gn0WPClI", + "1RAFXdMXgkXzaKtULuezGc7p1P1V3ZTwdLb7IdJscdqO5d2W9pb2LwQhQYq7K3hZs7V9BX+YDBNzz+OG", + "jMZVwxkC6tNLW1Tn9DJUZl14OWEt/IdKscV4Q0q7tD98O/wvAAD//8+zA+A+OwAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/api/queryService/job.go b/api/queryService/job.go index dd1479b1..57cc49d2 100644 --- a/api/queryService/job.go +++ b/api/queryService/job.go @@ -26,15 +26,16 @@ func (s *Controllers) CreateJob(ctx echo.Context) error { } func (s *Controllers) GetJob(ctx echo.Context, id types.UUID) error { - job, err := s.svc.Job.Get(ctx.Request().Context(), id) + job, err := s.svc.Job.GetWithStatus(ctx.Request().Context(), id) if err != nil { return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Unable to find job: %s", err)) } return ctx.JSON(http.StatusOK, Job{ - Id: job.ID, - ClientId: job.ClientID, - CanSync: job.CanSync, + Id: job.Job.ID, + ClientId: job.Job.ClientID, + CanSync: job.Job.CanSync, + Status: parseJobStatus(job.Status), }) } diff --git a/api/queryService/job_test.go b/api/queryService/job_test.go index 4bb90cdb..490100b1 100644 --- a/api/queryService/job_test.go +++ b/api/queryService/job_test.go @@ -57,22 +57,17 @@ func TestCreateJob(t *testing.T) { ctx := e.NewContext(req, rec) id := uuid.New() - collId := uuid.New() pool.ExpectQuery("name: CreateJob :one").WithArgs(database.MustToDBUUID(body.ClientId)).WillReturnRows( pgxmock.NewRows([]string{"id"}). AddRow(database.MustToDBUUID(id)), ) pool.ExpectBeginTx(pgx.TxOptions{}) - pool.ExpectQuery("name: CreateCollector :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows( - pgxmock.NewRows([]string{"id"}). - AddRow(database.MustToDBUUID(collId)), - ) - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(collId)).WillReturnRows( + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows( pgxmock.NewRows([]string{"version"}). AddRow(int32(1)), ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(collId), int32(1)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() @@ -117,6 +112,10 @@ func TestGetJob(t *testing.T) { pgxmock.NewRows([]string{"id", "name", "canSync"}). AddRow(database.MustToDBUUID(j.ClientID), "client_name", true), ) + pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows( + pgxmock.NewRows([]string{"issynced"}). + AddRow(true), + ) err = cons.GetJob(ctx, j.ID) assert.NoError(t, err) @@ -129,6 +128,7 @@ func TestGetJob(t *testing.T) { Id: j.ID, ClientId: j.ClientID, CanSync: j.CanSync, + Status: queryservice.INSYNC, }, res) } diff --git a/api/queryService/jobcollector_test.go b/api/queryService/jobcollector_test.go index edde9e7f..0aa9f904 100644 --- a/api/queryService/jobcollector_test.go +++ b/api/queryService/jobcollector_test.go @@ -44,7 +44,6 @@ func TestUpdateJobCollector(t *testing.T) { cfg.QueueClient = mockSQS current := collector.Collector{ - ID: uuid.New(), JobID: uuid.New(), ActiveVersion: 1, LatestVersion: 4, @@ -84,8 +83,8 @@ func TestUpdateJobCollector(t *testing.T) { pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(current.JobID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(current.ID), database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")), + pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). + AddRow(database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")), ) pool.ExpectQuery("name: AllQueriesExist :one").WithArgs([]pgtype.UUID{database.MustToDBUUID((*body.Fields)[0].QueryId)}). WillReturnRows( @@ -93,15 +92,15 @@ func TestUpdateJobCollector(t *testing.T) { AddRow(true), ) pool.ExpectBeginTx(pgx.TxOptions{}) - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.ID)).WillReturnRows( + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.JobID)).WillReturnRows( pgxmock.NewRows([]string{"version"}). AddRow(int32(5)), ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ID), int32(2)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.ID), int32(5), *body.MinimumCleanerVersion). + pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(5), *body.MinimumCleanerVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ID), "a", database.MustToDBUUID((*body.Fields)[0].QueryId), int32(5)). + pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.JobID), "a", database.MustToDBUUID((*body.Fields)[0].QueryId), int32(5)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() @@ -144,7 +143,6 @@ func TestGetJobCollectorByJobId(t *testing.T) { }) coll := collector.Collector{ - ID: uuid.New(), JobID: uuid.New(), MinCleanVersion: int32(1), MinTextVersion: int32(2), @@ -152,8 +150,8 @@ func TestGetJobCollectorByJobId(t *testing.T) { pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(coll.JobID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(coll.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")), + pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). + AddRow(database.MustToDBUUID(coll.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")), ) err = cons.GetJobCollectorByJobId(ctx, coll.JobID) diff --git a/api/queryService/parse.go b/api/queryService/parse.go index 6f01759c..d65f3255 100644 --- a/api/queryService/parse.go +++ b/api/queryService/parse.go @@ -2,6 +2,7 @@ package queryservice import ( "errors" + "queryorchestration/internal/job" "queryorchestration/internal/query" resultprocessor "queryorchestration/internal/query/result/processor" @@ -84,3 +85,12 @@ func parseStringToUUIDArray(sids *[]string) (*[]uuid.UUID, error) { return nil, nil } + +func parseJobStatus(status job.Status) JobStatus { + switch status { + case job.IN_SYNC: + return INSYNC + } + + return NOTSYNCED +} diff --git a/api/queryService/parse_test.go b/api/queryService/parse_test.go index 562e066e..fbd36497 100644 --- a/api/queryService/parse_test.go +++ b/api/queryService/parse_test.go @@ -1,6 +1,7 @@ package queryservice import ( + "queryorchestration/internal/job" "queryorchestration/internal/query" resultprocessor "queryorchestration/internal/query/result/processor" "testing" @@ -124,3 +125,14 @@ func TestParseStringToUUIDArray(t *testing.T) { assert.NoError(t, err) assert.Nil(t, out) } + +func TestParseSpecJobStatus(t *testing.T) { + qt := parseJobStatus(job.IN_SYNC) + assert.Equal(t, INSYNC, qt) + + qt = parseJobStatus(job.NOT_SYNCED) + assert.Equal(t, NOTSYNCED, qt) + + qt = parseJobStatus("invalid") + assert.Equal(t, NOTSYNCED, qt) +} diff --git a/api/queryService/query_test.go b/api/queryService/query_test.go index 053d7444..649e9135 100644 --- a/api/queryService/query_test.go +++ b/api/queryService/query_test.go @@ -11,7 +11,6 @@ import ( "queryorchestration/internal/document" cleanversion "queryorchestration/internal/document/clean/version" textversion "queryorchestration/internal/document/text/version" - "queryorchestration/internal/job" "queryorchestration/internal/job/collector" "queryorchestration/internal/query" "queryorchestration/internal/query/result" @@ -255,17 +254,8 @@ func TestTestQuery(t *testing.T) { }), }) - j := job.Job{ - ID: uuid.New(), - } - coll := collector.Collector{ - ID: uuid.New(), - JobID: j.ID, - } doc := document.Document{ - ID: uuid.New(), - JobID: coll.JobID, - Hash: "example_hash", + ID: uuid.New(), } params := &result.Process{ QueryID: uuid.New(), @@ -286,28 +276,19 @@ func TestTestQuery(t *testing.T) { ctx := e.NewContext(req, rec) reqID := database.MustToDBUUID(uuid.New()) - pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "hash"}). - AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash), - ) - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(doc.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")), - ) pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), ¶ms.QueryVersion).WillReturnRows( pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). AddRow(database.MustToDBUUID(params.QueryID), repository.QuerytypeJsonExtractor, int32(1), params.QueryVersion+1, []byte("{\"path\":\"oldkey\"}"), []pgtype.UUID{reqID}), ) - pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(params.QueryID), params.QueryVersion). + strVal := "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}" + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.QueryID), ¶ms.QueryVersion, database.MustToDBUUID(params.DocumentID)). WillReturnRows( - pgxmock.NewRows([]string{"queryId", "type", "value"}). - AddRow(reqID, repository.QuerytypeContextFull, "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}"), + pgxmock.NewRows([]string{"id", "queryId", "type", "value"}). + AddRow(database.MustToDBUUID(uuid.New()), reqID, repository.QuerytypeContextFull, &strVal), ) - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion).WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte("{\"path\":\"oldkey\"}")), + pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID)).WillReturnRows( + pgxmock.NewRows([]string{"config"}). + AddRow([]byte("{\"path\":\"oldkey\"}")), ) err = cons.TestQuery(ctx, params.QueryID) diff --git a/build/Dockerfile b/build/Dockerfile index 566a8182..9521c343 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -16,11 +16,7 @@ ARG TARGETARCH SHELL ["/bin/bash", "-c"] RUN --mount=type=cache,target=/go/pkg/mod/ \ --mount=type=bind,target=. \ - while read -r dir; do \ - CGO_ENABLED=0 GOARCH=$TARGETARCH go build -o "/bin/$(basename "$dir")" "$dir"; \ - done < <(find ./cmd -type d -not -name "*_test" -mindepth 1 -maxdepth 1) - - + CGO_ENABLED=0 GOARCH=$TARGETARCH go build -o /bin ./cmd/... FROM scratch AS final diff --git a/database/migrations/00000000000005_collectors.up.sql b/database/migrations/00000000000005_collectors.up.sql index 28b1f04f..bffa4323 100644 --- a/database/migrations/00000000000005_collectors.up.sql +++ b/database/migrations/00000000000005_collectors.up.sql @@ -1,16 +1,9 @@ -CREATE TABLE collectors ( - id uuid primary key DEFAULT uuid_generate_v7(), - jobId uuid not null, - foreign key (jobId) references jobs(id), - unique (jobId) -); - CREATE TABLE collectorVersions ( - collectorId uuid not null, + jobId uuid not null, id int not null, addedAt timestamp not null default current_timestamp, - primary key (id, collectorId), - foreign key (collectorId) references collectors(id) + foreign key (jobId) references jobs(id), + primary key (id, jobId) ); CREATE OR REPLACE FUNCTION setCollectorVersionNumber() @@ -19,8 +12,8 @@ BEGIN SELECT COALESCE(MAX(id), 0) + 1 INTO NEW.id FROM collectorVersions - WHERE collectorId = NEW.collectorId; - + WHERE jobId = NEW.jobId; + RETURN NEW; END; $$ LANGUAGE plpgsql; @@ -32,20 +25,20 @@ EXECUTE FUNCTION setCollectorVersionNumber(); CREATE TABLE collectorActiveVersions ( id uuid primary key DEFAULT uuid_generate_v7(), - collectorId uuid not null, + jobId uuid not null, versionId int not null, - foreign key (collectorId, versionId) references collectorVersions(collectorId, id) + foreign key (jobId, versionId) references collectorVersions(jobId, id) ); CREATE TABLE collectorMinCleanVersions ( id uuid primary key DEFAULT uuid_generate_v7(), - collectorId uuid not null, + jobId uuid not null, versionId int not null, addedVersion int not null, removedVersion int, - foreign key (collectorId, addedVersion) references collectorVersions(collectorId, id), - foreign key (collectorId, removedVersion) references collectorVersions(collectorId, id), - foreign key (collectorId) references collectors(id) + foreign key (jobId, addedVersion) references collectorVersions(jobId, id), + foreign key (jobId, removedVersion) references collectorVersions(jobId, id), + foreign key (jobId) references jobs(id) ); CREATE OR REPLACE FUNCTION removeMinCleanVersion() @@ -53,8 +46,8 @@ RETURNS TRIGGER AS $$ BEGIN UPDATE collectorMinCleanVersions SET removedVersion = NEW.addedVersion - WHERE collectorId = NEW.collectorId and removedVersion is null; - + WHERE jobId = NEW.jobId and removedVersion is null; + RETURN NEW; END; $$ LANGUAGE plpgsql; @@ -66,13 +59,13 @@ EXECUTE FUNCTION removeMinCleanVersion(); CREATE TABLE collectorMinTextVersions ( id uuid primary key DEFAULT uuid_generate_v7(), - collectorId uuid not null, + jobId uuid not null, versionId int not null, addedVersion int not null, removedVersion int, - foreign key (collectorId, addedVersion) references collectorVersions(collectorId, id), - foreign key (collectorId, removedVersion) references collectorVersions(collectorId, id), - foreign key (collectorId) references collectors(id) + foreign key (jobId, addedVersion) references collectorVersions(jobId, id), + foreign key (jobId, removedVersion) references collectorVersions(jobId, id), + foreign key (jobId) references jobs(id) ); CREATE OR REPLACE FUNCTION removeMinTextVersion() @@ -80,8 +73,8 @@ RETURNS TRIGGER AS $$ BEGIN UPDATE collectorMinTextVersions SET removedVersion = NEW.addedVersion - WHERE collectorId = NEW.collectorId and removedVersion is null; - + WHERE jobId = NEW.jobId and removedVersion is null; + RETURN NEW; END; $$ LANGUAGE plpgsql; @@ -93,14 +86,14 @@ EXECUTE FUNCTION removeMinTextVersion(); CREATE TABLE collectorQueries ( id uuid primary key DEFAULT uuid_generate_v7(), - collectorId uuid not null, + jobId uuid not null, name varchar(255) not null, queryId uuid not null, addedVersion int not null, removedVersion int, foreign key (queryId) references queries(id), - foreign key (collectorId) references collectors(id), - foreign key (collectorId, addedVersion) references collectorVersions(collectorId, id), - foreign key (collectorId, removedVersion) references collectorVersions(collectorId, id), - unique (collectorId, name, removedVersion) -); \ No newline at end of file + foreign key (jobId) references jobs(id), + foreign key (jobId, addedVersion) references collectorVersions(jobId, id), + foreign key (jobId, removedVersion) references collectorVersions(jobId, id), + unique (jobId, name, removedVersion) +); diff --git a/database/migrations/00000000000006_documents.up.sql b/database/migrations/00000000000006_documents.up.sql index 1f04281a..7d7c1a91 100644 --- a/database/migrations/00000000000006_documents.up.sql +++ b/database/migrations/00000000000006_documents.up.sql @@ -25,9 +25,9 @@ CREATE TABLE documentCleans ( CREATE TABLE documentTextExtractions ( id uuid primary key DEFAULT uuid_generate_v7(), - documentId uuid not null, + cleanEntryId uuid not null, version int not null, bucket text not null, key text not null, - foreign key (documentId) references documents(id) + foreign key (cleanEntryId) references documentCleans(id) ); diff --git a/database/migrations/00000000000007_results.up.sql b/database/migrations/00000000000007_results.up.sql index b9fbd222..e41ff5cf 100644 --- a/database/migrations/00000000000007_results.up.sql +++ b/database/migrations/00000000000007_results.up.sql @@ -1,13 +1,18 @@ CREATE TABLE results ( id uuid primary key DEFAULT uuid_generate_v7(), + textEntryId uuid not null, queryId uuid not null, - documentId uuid not null, value TEXT not null, - cleanVersion int not null, - textVersion int not null, queryVersion int not null, foreign key (queryId) references queries(id), - foreign key (documentId) references documents(id), - unique (queryId, documentId, cleanVersion, textVersion, queryVersion), + foreign key (textEntryId) references documentTextExtractions(id), foreign key (queryId, queryVersion) references queryVersions(queryId, id) -); \ No newline at end of file +); + +CREATE TABLE resultDependencies ( + resultId uuid not null, + requiredResultId uuid not null, + foreign key (resultId) references results(id), + foreign key (requiredResultId) references results(id), + CONSTRAINT result_not_self_dependent CHECK (resultId != requiredResultId) +); diff --git a/database/migrations/00000000000100_query_views.up.sql b/database/migrations/00000000000100_query_views.up.sql index ad39d94f..348a5889 100644 --- a/database/migrations/00000000000100_query_views.up.sql +++ b/database/migrations/00000000000100_query_views.up.sql @@ -1,59 +1,71 @@ CREATE VIEW queryCurrentActiveVersions as SELECT DISTINCT - av.queryId, - (FIRST_VALUE(av.versionId) OVER (PARTITION BY av.queryId ORDER BY av.id DESC))::int as id - FROM queryActiveVersions as av; + q.id as queryId, + coalesce( + (FIRST_VALUE(av.versionId) OVER (PARTITION BY q.id ORDER BY av.id DESC)), + 0 + )::int as activeVersion + FROM queries AS q + LEFT JOIN queryActiveVersions as av on av.queryId = q.id; CREATE VIEW queryLatestVersions as - SELECT queryId, max(id)::int as id - FROM queryVersions - GROUP BY queryId; + SELECT + q.id as queryId, + coalesce(max(v.id), 0)::int as latestVersion + FROM queries AS q + LEFT JOIN queryVersions as v on v.queryId = q.id + GROUP BY q.id; -CREATE VIEW fullActiveQueries AS -with config as ( -SELECT c.queryId, c.config - FROM queries AS q - LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId - LEFT JOIN queryConfigs AS c ON q.id = c.queryId - and isInVersion(av.id, c.addedVersion, c.removedVersion) -), -requiredIds as ( - SELECT r.queryId, - ARRAY_AGG(DISTINCT r.requiredQueryId) - FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[] - as requiredIds - FROM queries AS q - LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId - LEFT JOIN requiredQueries AS r ON q.id = r.queryId - and isInVersion(av.id, r.addedVersion, r.removedVersion) - GROUP BY r.queryId -) -SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config, +CREATE VIEW queryCurrentConfigs as + SELECT av.queryId, c.config + FROM queryCurrentActiveVersions as av + LEFT JOIN queryConfigs AS c ON av.queryId = c.queryId + and isInVersion(av.activeVersion, c.addedVersion, c.removedVersion); + +CREATE VIEW queryCurrentRequiredIds as + SELECT DISTINCT av.queryId, r.requiredQueryId + FROM queryCurrentActiveVersions as av + LEFT JOIN requiredQueries AS r ON av.queryId = r.queryId + and isInVersion(av.activeVersion, r.addedVersion, r.removedVersion); + +CREATE VIEW queryCurrentRequiredIdsAGG as + SELECT queryId, coalesce( - r.requiredIds, + ARRAY_AGG(DISTINCT requiredQueryId) + FILTER (WHERE requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[], array[]::uuid[] )::uuid[] as requiredIds + FROM queryCurrentRequiredIds + GROUP BY queryId; + +CREATE VIEW fullActiveQueries AS +SELECT DISTINCT q.id, q.type, av.activeVersion, lv.latestVersion, c.config, r.requiredIds FROM queries AS q - LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId - LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id - LEFT JOIN config AS c ON q.id = c.queryId - LEFT JOIN requiredIds AS r ON q.id = r.queryId; + JOIN queryCurrentActiveVersions as av on q.id = av.queryId + JOIN queryLatestVersions as lv on lv.queryId = q.id + JOIN queryCurrentConfigs AS c ON q.id = c.queryId + JOIN queryCurrentRequiredIdsAGG AS r ON q.id = r.queryId; CREATE VIEW queryActiveDependencies AS - with recursive queryActiveDependencies AS ( - SELECT q.id, r.requiredQueryId - FROM queries AS q - LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId - LEFT JOIN requiredQueries AS r ON q.id = r.queryId - and isInVersion(av.id, r.addedVersion, r.removedVersion) +WITH RECURSIVE queryActiveDependencies(queryId, requiredQueryId, path, cycle) AS ( + SELECT + queryId, + requiredQueryId, + ARRAY[queryId, requiredQueryId]::uuid[] AS path, + false AS cycle + FROM queryCurrentRequiredIds UNION ALL - SELECT qd.id, r.requiredQueryId - FROM queries AS q - LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId - LEFT JOIN requiredQueries AS r ON q.id = r.queryId - and isInVersion(av.id, r.addedVersion, r.removedVersion) - JOIN queryActiveDependencies as qd on q.id = qd.requiredQueryId + SELECT + q.queryId, + qd.requiredQueryId, + path || qd.requiredQueryId, + qd.requiredQueryId = ANY(path) AS cycle + FROM queryCurrentRequiredIds as q + JOIN queryActiveDependencies as qd ON q.queryId = qd.requiredQueryId + WHERE NOT qd.cycle -- Stop if we detect a cycle ) - SELECT DISTINCT id, requiredQueryId FROM queryActiveDependencies; \ No newline at end of file + SELECT DISTINCT queryId as id, requiredQueryId + FROM queryActiveDependencies + WHERE NOT cycle; diff --git a/database/migrations/00000000000101_collector_views.up.sql b/database/migrations/00000000000101_collector_views.up.sql index 58fcb40c..cb5071e5 100644 --- a/database/migrations/00000000000101_collector_views.up.sql +++ b/database/migrations/00000000000101_collector_views.up.sql @@ -1,47 +1,77 @@ CREATE VIEW collectorCurrentActiveVersions as SELECT DISTINCT - av.collectorId, - (FIRST_VALUE(av.versionId) OVER (PARTITION BY av.collectorId ORDER BY av.id DESC))::int as id - FROM collectorActiveVersions as av; + j.id as jobId, + coalesce( + (FIRST_VALUE(v.versionId) OVER (PARTITION BY j.id ORDER BY v.id DESC)), + 0 + )::int as activeVersion + FROM jobs as j + LEFT JOIN collectorActiveVersions as v on v.jobId = j.id; CREATE VIEW collectorLatestVersions as - SELECT collectorId, max(id)::int as id - FROM collectorVersions - GROUP BY collectorId; + SELECT + j.id as jobId, + coalesce(max(v.id), 0)::int as latestVersion + FROM jobs as j + LEFT JOIN collectorVersions as v on v.jobId = j.id + GROUP BY j.id; + +CREATE VIEW currentCollectorMinTextVersions as +SELECT DISTINCT + av.jobId, + coalesce(ctv.versionId, 0) as minTextVersion + FROM collectorCurrentActiveVersions as av + LEFT JOIN collectorMinTextVersions AS ctv ON av.jobId = ctv.jobId + and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion); + +CREATE VIEW currentCollectorMinCleanVersions as +SELECT DISTINCT + av.jobId, + coalesce(ctv.versionId, 0) as minCleanVersion + FROM collectorCurrentActiveVersions as av + LEFT JOIN collectorMinCleanVersions AS ctv ON av.jobId = ctv.jobId + and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion); + +CREATE VIEW currentCollectorQueries as +SELECT DISTINCT + av.jobId, + q.name, + q.queryId + FROM collectorCurrentActiveVersions as av + LEFT JOIN collectorQueries AS q ON av.jobId = q.jobId + and isInVersion(av.activeVersion, q.addedVersion, q.removedVersion); + +CREATE VIEW currentCollectorQueriesJSONAGG as +SELECT DISTINCT + jobId, + jsonb_object_agg(name, queryId) FILTER (WHERE name is not null) AS fields + FROM currentCollectorQueries + GROUP BY jobId; CREATE VIEW fullActiveCollectors AS - SELECT DISTINCT c.id, c.jobId, coalesce(ccv.versionId, 0) as minCleanVersion, coalesce(ctv.versionId, 0) as minTextVersion, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, - jsonb_object_agg(q.name, q.queryId) FILTER (WHERE q.name is not null) AS fields - FROM collectors AS c - LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId - LEFT JOIN collectorLatestVersions as lv on lv.collectorId = c.id - LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId - and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) - LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId - and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion) - LEFT JOIN collectorQueries AS q ON c.id = q.collectorId - and isInVersion(av.id, q.addedVersion, q.removedVersion) - GROUP BY c.id, c.jobId, ccv.versionId, ctv.versionId, av.id, lv.id; + SELECT DISTINCT av.jobId, + ccv.minCleanVersion, ctv.minTextVersion, + av.activeVersion, lv.latestVersion, + q.fields + FROM collectorCurrentActiveVersions as av + JOIN collectorLatestVersions as lv on lv.jobId = av.jobId + JOIN currentCollectorMinCleanVersions AS ccv ON av.jobId = ccv.jobId + JOIN currentCollectorMinTextVersions AS ctv ON av.jobId = ctv.jobId + JOIN currentCollectorQueriesJSONAGG AS q ON av.jobId = q.jobId; CREATE VIEW collectorQueryDependencyTree AS - WITH RECURSIVE - coll as ( - SELECT DISTINCT c.id, c.jobId, q.queryId - FROM collectors as c - LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId - LEFT JOIN collectorQueries as q ON c.id = q.collectorId - and isInVersion(av.id, q.addedVersion, q.removedVersion) - ), - collectorQueryDependencyTree AS ( - SELECT cq.id as collectorId, cq.jobId, aqc.id as queryId, aqc.type, aqc.requiredIds, aqc.activeVersion - FROM coll as cq - JOIN fullActiveQueries as aqc on cq.queryId = aqc.id + WITH RECURSIVE collectorQueryDependencyTree AS ( + SELECT cq.jobId, cq.queryId, ri.requiredIds + FROM currentCollectorQueries as cq + JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId UNION ALL - SELECT acq.collectorId, acq.jobId, q.id as queryId, q.type, q.requiredIds, q.activeVersion - FROM fullActiveQueries as q - JOIN collectorQueryDependencyTree as acq on q.id = ANY(acq.requiredIds) + SELECT acq.jobId, q.queryId, q.requiredIds + FROM queryCurrentRequiredIdsAGG as q + JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds) ) - SELECT DISTINCT collectorId, jobId, queryId, type, activeVersion as queryVersion, requiredIds - FROM collectorQueryDependencyTree; \ No newline at end of file + SELECT DISTINCT ct.jobId, ct.queryId, q.type, av.activeVersion as queryVersion, ct.requiredIds + FROM collectorQueryDependencyTree as ct + JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId + JOIN queries as q on q.id = ct.queryId; diff --git a/database/migrations/00000000000102_job_views.up.sql b/database/migrations/00000000000102_job_views.up.sql index 1deb774a..52081b91 100644 --- a/database/migrations/00000000000102_job_views.up.sql +++ b/database/migrations/00000000000102_job_views.up.sql @@ -8,7 +8,7 @@ DECLARE totalCount BIGINT := 0; BEGIN SELECT COUNT(*)::BIGINT INTO totalCount FROM documents WHERE jobId = _jobId; - + RETURN QUERY SELECT d.id, totalCount FROM documents as d @@ -17,4 +17,83 @@ BEGIN LIMIT _batchSize OFFSET _offset; END; -$$ LANGUAGE plpgsql; \ No newline at end of file +$$ LANGUAGE plpgsql; + +CREATE VIEW currentCleanEntries as +WITH RankedExtractions AS ( + SELECT + dc.id, + dc.documentId, + dc.bucket, + dc.key, + dc.version, + ROW_NUMBER() OVER (PARTITION BY dc.documentId ORDER BY dc.id DESC) as row_num + FROM documents d + JOIN currentCollectorMinCleanVersions ccv ON d.jobId = ccv.jobId + JOIN documentCleans dc ON dc.documentId = d.id + AND dc.version >= ccv.minCleanVersion +) +SELECT + id, + documentId, + bucket, + key, + version +FROM RankedExtractions +WHERE row_num = 1; + +CREATE VIEW currentTextEntries as +WITH RankedExtractions AS ( + SELECT + dte.id, + cc.documentId, + dte.bucket, + dte.key, + dte.version, + dte.cleanEntryId, + ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dte.id DESC) as row_num + FROM documents d + JOIN currentCleanEntries cc on cc.documentId = d.id + JOIN currentCollectorMinTextVersions ctv ON ctv.jobId = d.jobId + JOIN documentTextExtractions dte ON dte.cleanEntryId = cc.id + AND dte.version >= ctv.minTextVersion +) +SELECT + id, + documentId, + bucket, + key, + version, + cleanEntryId +FROM RankedExtractions +WHERE row_num = 1; + +CREATE VIEW currentClientCanSync as +WITH RankedExtractions AS ( + SELECT + ccs.clientId, + ccs.canSync, + ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.id DESC) as row_num + FROM clients c + JOIN clientCanSync ccs on c.id = ccs.clientId +) +SELECT + c.id as clientId, + coalesce(r.canSync, false) as canSync +FROM clients c +LEFT JOIN RankedExtractions r on r.clientId = c.id and r.row_num = 1; + +CREATE VIEW currentJobCanSync as +WITH RankedExtractions AS ( + SELECT + ccs.jobId, + ccs.canSync, + ROW_NUMBER() OVER (PARTITION BY ccs.jobId ORDER BY ccs.id DESC) as row_num + FROM jobs c + JOIN jobCanSync ccs on c.id = ccs.jobId +) +SELECT + c.id as jobId, + coalesce(r.canSync, false) as canSync +FROM jobs c +LEFT JOIN RankedExtractions r on r.jobId = c.id and r.row_num = 1; diff --git a/database/queries/clean.sql b/database/queries/clean.sql index 71dacdad..445ac191 100644 --- a/database/queries/clean.sql +++ b/database/queries/clean.sql @@ -1,25 +1,12 @@ -- name: IsDocumentClean :one SELECT EXISTS( - SELECT 1 - FROM documentCleans AS dc - JOIN documents AS d ON d.id = dc.documentId - LEFT JOIN collectors as c ON d.jobId = c.jobId - LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId - LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId - and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) - WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1) + SELECT 1 FROM currentCleanEntries WHERE documentId = $1 ); -- name: AddDocumentCleanEntry :exec INSERT INTO documentCleans (documentId, version, bucket, key) VALUES ($1, $2, $3, $4); -- name: GetDocumentCleanEntry :one -SELECT dc.documentId, dc.bucket, dc.key, dc.version - FROM documentCleans AS dc - JOIN documents AS d ON d.id = dc.documentId - LEFT JOIN collectors as c ON d.jobId = c.jobId - LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId - LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId - and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) - WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1) - ORDER BY d.id DESC LIMIT 1; \ No newline at end of file +SELECT id, documentId, bucket, key, version + FROM currentCleanEntries + WHERE documentId = $1; \ No newline at end of file diff --git a/database/queries/client.sql b/database/queries/client.sql index e0ddc2cb..64449071 100644 --- a/database/queries/client.sql +++ b/database/queries/client.sql @@ -2,15 +2,9 @@ INSERT INTO clients (name) VALUES ($1) RETURNING id; -- name: GetClient :one -SELECT c.id, c.name, coalesce(cs.canSync, false) as canSync +SELECT c.id, c.name, cs.canSync FROM clients as c - LEFT JOIN ( - SELECT clientId, canSync - FROM clientCanSync - WHERE clientId = $1 - ORDER BY id DESC - LIMIT 1 - ) as cs on cs.clientId = c.id + JOIN currentClientCanSync as cs on cs.clientId = c.id WHERE c.id = $1; -- name: UpdateClient :exec diff --git a/database/queries/collector.sql b/database/queries/collector.sql index c72ab911..3e68f7be 100644 --- a/database/queries/collector.sql +++ b/database/queries/collector.sql @@ -1,29 +1,26 @@ -- name: ListCollectorQueries :many -SELECT * FROM collectorQueryDependencyTree WHERE collectorId = $1; +SELECT * FROM collectorQueryDependencyTree WHERE jobId = @jobId; -- name: GetCollectorByJobID :one -SELECT * FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1; +SELECT * FROM fullActiveCollectors WHERE jobId = @jobId LIMIT 1; -- name: GetCollector :one -SELECT * FROM fullActiveCollectors WHERE id = $1 LIMIT 1; - --- name: CreateCollector :one -INSERT INTO collectors (jobId) VALUES ($1) RETURNING id; +SELECT * FROM fullActiveCollectors WHERE jobId = @jobId LIMIT 1; -- name: AddLatestCollectorVersion :one -INSERT INTO collectorVersions (collectorId) VALUES ($1) RETURNING id; +INSERT INTO collectorVersions (jobId) VALUES ($1) RETURNING id; -- name: SetActiveCollectorVersion :exec -INSERT INTO collectorActiveVersions (collectorId, versionId) VALUES ($1, $2); +INSERT INTO collectorActiveVersions (jobId, versionId) VALUES ($1, $2); -- name: SetCollectorCleanVersion :exec -INSERT INTO collectorMinCleanVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3); +INSERT INTO collectorMinCleanVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3); -- name: SetCollectorTextVersion :exec -INSERT INTO collectorMinTextVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3); +INSERT INTO collectorMinTextVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3); -- name: AddCollectorQuery :exec -INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4); +INSERT INTO collectorQueries (jobId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4); -- name: RemoveCollectorQuery :exec -UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and collectorId = $3 and removedVersion is null; +UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and jobId = $3 and removedVersion is null; diff --git a/database/queries/job.sql b/database/queries/job.sql index d64008fa..8333edd0 100644 --- a/database/queries/job.sql +++ b/database/queries/job.sql @@ -1,13 +1,7 @@ -- name: GetJob :one -SELECT j.id, j.clientId, coalesce(cs.canSync, false) as canSync +SELECT j.id, j.clientId, cs.canSync FROM jobs as j - LEFT JOIN ( - SELECT jobId, canSync - FROM jobCanSync - WHERE jobId = $1 - ORDER BY id DESC - LIMIT 1 - ) as cs on cs.jobId = j.id + JOIN currentJobCanSync as cs on cs.jobId = j.id WHERE j.id = $1; -- name: CreateJob :one @@ -18,3 +12,79 @@ INSERT INTO jobCanSync (canSync, jobId) VALUES ($1, $2); -- name: ListJobDocumentIDsBatch :many SELECT id, totalCount FROM listJobDocumentIDs(@jobId, @batchSize, @pageOffset); + +-- name: IsJobSynced :one +WITH +docs AS ( + -- Get all documents for this job + SELECT id, jobId FROM documents WHERE jobId = $1 +), +doc_text_entries AS ( + -- Documents with their current text entries + SELECT + d.id AS document_id, + cte.id AS text_entry_id + FROM + docs d + LEFT JOIN currentTextEntries cte ON cte.documentId = d.id +), +required_results AS ( + -- All required document-query-version combinations + SELECT + d.id AS document_id, + cqdt.queryId, + cqdt.queryVersion, + dte.text_entry_id + FROM + docs d + JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.jobId + JOIN doc_text_entries dte ON dte.document_id = d.id + and dte.text_entry_id IS NOT NULL +), +existing_results AS ( + -- Valid results that exist + SELECT + rr.document_id AS document_id, + r.queryId, + r.id AS result_id + FROM + results r + JOIN required_results rr ON + r.queryId = rr.queryId AND + r.queryVersion = rr.queryVersion AND + r.textEntryId = rr.text_entry_id +), +missing_results AS ( + -- Find missing results + SELECT rr.queryId + FROM required_results rr + LEFT JOIN existing_results er on er.queryId = rr.queryId + WHERE er.result_id is null +), +dependency_check AS ( + -- Check for missing dependencies + SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId + FROM existing_results er + JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId + LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id + WHERE rd.resultId is null +) +SELECT ( + -- No documents means job is synced + NOT EXISTS (SELECT 1 FROM docs) + + OR + + -- Documents with no text entries + (NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL) + + and + + -- Documents with missing results + NOT EXISTS (SELECT 1 FROM missing_results) + + and + + -- Documents with missing dependencies + NOT EXISTS (SELECT 1 FROM dependency_check)) +)::bool as is_synced; diff --git a/database/queries/query.sql b/database/queries/query.sql index 1a3bdede..d9af8ea9 100644 --- a/database/queries/query.sql +++ b/database/queries/query.sql @@ -1,5 +1,5 @@ --- name: GetQueryConfig :one -SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 and COALESCE(removedVersion, $2 - 1) < $2; +-- name: GetActiveQueryConfig :one +SELECT config FROM queryCurrentConfigs where queryId = $1; -- name: GetQuery :one SELECT * FROM fullActiveQueries WHERE id = $1; @@ -16,22 +16,23 @@ SELECT c.queryId, c.config ), requiredIds as ( SELECT r.queryId, + coalesce( ARRAY_AGG(DISTINCT r.requiredQueryId) - FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[] - as requiredIds + FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[], + array[]::uuid[] + )::uuid[] as requiredIds FROM query AS q LEFT JOIN requiredQueries AS r ON q.id = r.queryId and isInVersion(@version, r.addedVersion, r.removedVersion) GROUP BY r.queryId ) -SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config, - coalesce( - r.requiredIds, - array[]::uuid[] - )::uuid[] as requiredIds +SELECT DISTINCT q.id, q.type, + av.activeVersion, + lv.latestVersion, c.config, + r.requiredIds FROM query AS q - LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId - LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id + JOIN queryCurrentActiveVersions as av on q.id = av.queryId + JOIN queryLatestVersions as lv on lv.queryId = q.id LEFT JOIN config AS c ON q.id = c.queryId LEFT JOIN requiredIds AS r ON q.id = r.queryId; @@ -66,17 +67,21 @@ SELECT COUNT(*) = COUNT(DISTINCT id) AS all_exist -- name: IsQueryInDependencyTree :one SELECT EXISTS ( - SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1) + SELECT 1 FROM queryActiveDependencies + WHERE id = any(@requiredQueryIds) + and requiredQueryId = @queryId + or @queryId = any(@requiredQueryIds) ); -- name: ListQueryDirectDependentsByDocumentID :many WITH doc AS ( - SELECT id, jobId FROM documents where id = $2 + SELECT id, jobId FROM documents where id = @documentId ) SELECT dt.queryId - FROM collectorQueryDependencyTree as dt - JOIN doc as d on d.jobId = dt.jobId - where $1 = any(dt.requiredIds); + FROM doc as d + JOIN collectorQueryDependencyTree as dt + on d.jobId = dt.jobId + and @queryId = any(dt.requiredIds); -- name: ListQueryJobIDs :many -SELECT jobId FROM collectorQueryDependencyTree WHERE queryId = $1; \ No newline at end of file +SELECT jobId FROM collectorQueryDependencyTree WHERE queryId = $1; diff --git a/database/queries/result.sql b/database/queries/result.sql index 32cee5e0..6f810eee 100644 --- a/database/queries/result.sql +++ b/database/queries/result.sql @@ -1,9 +1,9 @@ -- name: ListQueryRequirementValues :many WITH reqQueries as ( - SELECT q.id as queryId, av.id as activeVersion, q.type + SELECT av.queryId, av.activeVersion, q.type FROM requiredQueries as rq - JOIN queries as q on q.id = rq.requiredQueryId - LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId + JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId + JOIN queries as q on q.id = av.queryId WHERE rq.queryId = @queryId and isInVersion(@version, rq.addedVersion, rq.removedVersion) ), @@ -13,70 +13,72 @@ docs as ( WHERE id = @documentId ), codeVersions as ( - SELECT + SELECT d.id as documentId, - coalesce(ccv.versionId, 1) as minCleanVersion, - coalesce(ctv.versionId, 1) as minTextVersion + mcv.minCleanVersion, + mtv.minTextVersion FROM docs as d - LEFT JOIN collectors as c on c.jobId = d.jobId - LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId - LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId - and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) - LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId - and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion) - LIMIT 1 + JOIN currentCollectorMinTextVersions as mtv on mtv.jobId = d.jobId + JOIN currentCollectorMinCleanVersions as mcv on mcv.jobId = d.jobId ), latestVersions AS ( - SELECT + SELECT + r.id, rq.queryId, rq.type, r.queryVersion, - r.cleanVersion, - r.textVersion, + r.textEntryId, + r.value, ROW_NUMBER() OVER ( - PARTITION BY rq.queryId - ORDER BY r.cleanVersion DESC, r.textVersion DESC + PARTITION BY r.queryId + ORDER BY r.id DESC ) as rowNumber FROM reqQueries as rq + JOIN currentTextEntries as cte on cte.documentId = @documentId LEFT JOIN results as r ON rq.queryId = r.queryId - and r.documentId = @documentId and r.queryVersion = rq.activeVersion - JOIN codeVersions as ccv on ccv.documentId = r.documentId - and r.cleanVersion >= ccv.minCleanVersion - and r.textVersion >= ccv.minTextVersion + and cte.id = r.textEntryId ) -SELECT DISTINCT lv.queryId, lv.type, r.value -FROM latestVersions as lv -JOIN results as r ON r.queryId = lv.queryId - and r.documentId = @documentId - and r.queryVersion = lv.queryVersion - and r.cleanVersion = lv.cleanVersion - and r.textVersion = lv.textVersion - and lv.rowNumber = 1; +SELECT DISTINCT id, queryId, type, value +FROM latestVersions +WHERE rowNumber = 1; --- name: SetResult :exec -INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6); +-- name: AddResult :one +INSERT INTO results (queryId, value, textEntryId, queryVersion) VALUES ($1, $2, $3, $4) returning id; + +-- name: AddResultDependency :exec +INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2); -- name: GetResultValueWithVersion :one -SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5; +WITH doc as ( + SELECT id, jobId + FROM documents + WHERE id = @documentId +) +SELECT r.id, r.value + FROM doc as d + JOIN currentTextEntries as cte on cte.documentId = d.id + LEFT JOIN results as r + on r.queryId = @queryId + and r.queryVersion = @queryVersion + and r.textEntryId = cte.id; -- name: ListUnsyncedNoDepsQueriesByDocId :many WITH docs as ( SELECT id, jobId from documents where id = $1 ), unsyncedQueries AS ( - SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id + SELECT DISTINCT dt.queryId, dt.requiredIds from docs as d JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId - JOIN fullActiveCollectors as c on c.id = dt.collectorId - LEFT JOIN results as r on r.queryId = dt.queryId - and r.documentId = d.id + JOIN currentTextEntries as cte on cte.documentId = d.id + LEFT JOIN results as r + on r.queryId = dt.queryId and r.queryVersion = dt.queryVersion - and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion + and cte.id = r.textEntryId where r.value is null ) SELECT DISTINCT queryId FROM unsyncedQueries as baseuq WHERE NOT EXISTS ( SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds) ); - \ No newline at end of file diff --git a/database/queries/text.sql b/database/queries/text.sql index d2bdfa37..6fe2bda7 100644 --- a/database/queries/text.sql +++ b/database/queries/text.sql @@ -1,25 +1,12 @@ -- name: IsDocumentTextExtracted :one SELECT EXISTS( - SELECT 1 - FROM documentTextExtractions AS dc - JOIN documents AS d ON d.id = dc.documentId - LEFT JOIN collectors as c ON d.jobId = c.jobId - LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId - LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId - and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) - WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1) + SELECT 1 FROM currentTextEntries WHERE documentId = @documentId ); -- name: AddDocumentTextEntry :exec -INSERT INTO documentTextExtractions (documentId, version, bucket, key) VALUES ($1, $2, $3, $4); +INSERT INTO documentTextExtractions (version, bucket, key, cleanEntryId) VALUES ($1, $2, $3, $4); -- name: GetDocumentTextEntry :one -SELECT dc.documentId, dc.bucket, dc.key, dc.version - FROM documentTextExtractions AS dc - JOIN documents AS d ON d.id = dc.documentId - LEFT JOIN collectors as c ON d.jobId = c.jobId - LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId - LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId - and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) - WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1) - ORDER BY d.id DESC LIMIT 1; \ No newline at end of file +SELECT id, documentId, bucket, key, version, cleanEntryId + FROM currentTextEntries + WHERE documentId = @documentId; diff --git a/devbox.json b/devbox.json index 1d2715ea..9acc17a8 100644 --- a/devbox.json +++ b/devbox.json @@ -1,5 +1,38 @@ { "$schema": "https://raw.githubusercontent.com/jetify-com/devbox/0.13.7/.schema/devbox.schema.json", + "env": { + "APP_ENV": "development", + "AWS_ACCESS_KEY_ID": "test", + "AWS_ENDPOINT_URL": "http://localhost:4566", + "AWS_REGION": "us-east-1", + "AWS_SECRET_ACCESS_KEY": "test", + "AWS_SESSION_TOKEN": "", + "BUCKET_IN": "documentin", + "DB_NOSSL": "true", + "DOCUMENT_CLEAN_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_clean", + "DOCUMENT_INIT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_init", + "DOCUMENT_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_sync", + "DOCUMENT_TEXT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_text", + "JOB_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/job_sync", + "PGDATABASE": "query_orchestration", + "PGHOST": "localhost", + "PGPASSWORD": "pass", + "PGPORT": "5432", + "PGPORT_GENERATE": "5431", + "PGUSER": "postgres", + "QNAME_DOCUMENT_CLEAN": "document_clean", + "QNAME_DOCUMENT_INIT": "document_init", + "QNAME_DOCUMENT_SYNC": "document_sync", + "QNAME_DOCUMENT_TEXT": "document_text", + "QNAME_JOB_SYNC": "job_sync", + "QNAME_QUERY_RUNNER": "query_runner", + "QNAME_QUERY_SYNC": "query_sync", + "QNAME_QUERY_VERSION_SYNC": "query_version_sync", + "QUERY_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_sync", + "QUERY_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_runner", + "QUERY_VERSION_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_version_sync" + }, + "env_from": ".env", "packages": [ "golangci-lint@1.62.2", "protolint@0.50.5", @@ -18,7 +51,8 @@ "oapi-codegen@2.4.1", "vacuum-go@0.14.1", "awscli2@2.19.0", - "postgresql@17.2" + "postgresql@17.2", + "bc@1.07.1" ], "shell": { "init_hook": [ @@ -26,38 +60,5 @@ "export DB_URI_GENERATE=postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT_GENERATE}/${PGDATABASE}?sslmode=disable", "echo 'Welcome to the DoczyAI devbox!'" ] - }, - "env": { - "APP_ENV": "development", - "PGUSER": "postgres", - "PGPASSWORD": "pass", - "PGHOST": "localhost", - "PGPORT": "5432", - "PGPORT_GENERATE": "5431", - "PGDATABASE": "query_orchestration", - "DB_NOSSL": "true", - "AWS_ACCESS_KEY_ID": "test", - "AWS_SECRET_ACCESS_KEY": "test", - "AWS_SESSION_TOKEN": "", - "AWS_REGION": "us-east-1", - "AWS_ENDPOINT_URL": "http://localhost:4566", - "QNAME_DOCUMENT_INIT": "document_init", - "QNAME_DOCUMENT_SYNC": "document_sync", - "QNAME_DOCUMENT_CLEAN": "document_clean", - "QNAME_DOCUMENT_TEXT": "document_text", - "QNAME_QUERY_SYNC": "query_sync", - "QNAME_QUERY_RUNNER": "query_runner", - "QNAME_JOB_SYNC": "job_sync", - "QNAME_QUERY_VERSION_SYNC": "query_version_sync", - "DOCUMENT_INIT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_init", - "DOCUMENT_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_sync", - "DOCUMENT_CLEAN_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_clean", - "DOCUMENT_TEXT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_text", - "QUERY_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_sync", - "QUERY_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_runner", - "JOB_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/job_sync", - "QUERY_VERSION_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_version_sync", - "BUCKET_IN": "documentin" - }, - "env_from": ".env" -} + } +} \ No newline at end of file diff --git a/devbox.lock b/devbox.lock index 4a644d45..fccc6521 100644 --- a/devbox.lock +++ b/devbox.lock @@ -65,34 +65,51 @@ } } }, - "github:NixOS/nixpkgs/nixpkgs-unstable": { - "resolved": "github:NixOS/nixpkgs/b1b43d32be000928cc71250ed77f4a0a5f2bc23a?lastModified=1739698114&narHash=sha256-8S9n69Dnpg8DhfFlP0YvMGmSOY2X4kImGSPWXYNpaHM%3D" - }, - "glibcLocales@latest": { + "bc@1.07.1": { "last_modified": "2025-02-07T11:26:36Z", - "resolved": "github:NixOS/nixpkgs/d98abf5cf5914e5e4e9d57205e3af55ca90ffc1d#glibcLocales", + "resolved": "github:NixOS/nixpkgs/d98abf5cf5914e5e4e9d57205e3af55ca90ffc1d#bc", "source": "devbox-search", - "version": "2.40-66", + "version": "1.07.1", "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/vbpa5cim632v5m8lkd3jx6gfmny1m6vf-bc-1.07.1", + "default": true + } + ], + "store_path": "/nix/store/vbpa5cim632v5m8lkd3jx6gfmny1m6vf-bc-1.07.1" + }, "aarch64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/msl6h7i0mgzayw4icb58hccfkh9672bl-glibc-locales-2.40-66", + "path": "/nix/store/49rm0gkxa9mrfnvzvgnfmcmwim3ilc16-bc-1.07.1", "default": true } ], - "store_path": "/nix/store/msl6h7i0mgzayw4icb58hccfkh9672bl-glibc-locales-2.40-66" + "store_path": "/nix/store/49rm0gkxa9mrfnvzvgnfmcmwim3ilc16-bc-1.07.1" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/4b3aqnqpnqd8j425f0isg4bfr56wiyfr-bc-1.07.1", + "default": true + } + ], + "store_path": "/nix/store/4b3aqnqpnqd8j425f0isg4bfr56wiyfr-bc-1.07.1" }, "x86_64-linux": { "outputs": [ { "name": "out", - "path": "/nix/store/p1fn0y9zbq7fng8cn49cvinqqh47slax-glibc-locales-2.40-66", + "path": "/nix/store/4bfbksfiy1b6r6yna9hh39qv8jfi33n7-bc-1.07.1", "default": true } ], - "store_path": "/nix/store/p1fn0y9zbq7fng8cn49cvinqqh47slax-glibc-locales-2.40-66" + "store_path": "/nix/store/4bfbksfiy1b6r6yna9hh39qv8jfi33n7-bc-1.07.1" } } }, diff --git a/internal/database/repository/clean.sql.go b/internal/database/repository/clean.sql.go index 014349fd..fcafc3b4 100644 --- a/internal/database/repository/clean.sql.go +++ b/internal/database/repository/clean.sql.go @@ -36,39 +36,21 @@ func (q *Queries) AddDocumentCleanEntry(ctx context.Context, arg *AddDocumentCle } const getDocumentCleanEntry = `-- name: GetDocumentCleanEntry :one -SELECT dc.documentId, dc.bucket, dc.key, dc.version - FROM documentCleans AS dc - JOIN documents AS d ON d.id = dc.documentId - LEFT JOIN collectors as c ON d.jobId = c.jobId - LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId - LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId - and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) - WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1) - ORDER BY d.id DESC LIMIT 1 +SELECT id, documentId, bucket, key, version + FROM currentCleanEntries + WHERE documentId = $1 ` -type GetDocumentCleanEntryRow struct { - Documentid pgtype.UUID `db:"documentid"` - Bucket string `db:"bucket"` - Key string `db:"key"` - Version int32 `db:"version"` -} - // GetDocumentCleanEntry // -// SELECT dc.documentId, dc.bucket, dc.key, dc.version -// FROM documentCleans AS dc -// JOIN documents AS d ON d.id = dc.documentId -// LEFT JOIN collectors as c ON d.jobId = c.jobId -// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId -// LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId -// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) -// WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1) -// ORDER BY d.id DESC LIMIT 1 -func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.UUID) (*GetDocumentCleanEntryRow, error) { +// SELECT id, documentId, bucket, key, version +// FROM currentCleanEntries +// WHERE documentId = $1 +func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.UUID) (*Currentcleanentry, error) { row := q.db.QueryRow(ctx, getDocumentCleanEntry, documentid) - var i GetDocumentCleanEntryRow + var i Currentcleanentry err := row.Scan( + &i.ID, &i.Documentid, &i.Bucket, &i.Key, @@ -79,28 +61,14 @@ func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.U const isDocumentClean = `-- name: IsDocumentClean :one SELECT EXISTS( - SELECT 1 - FROM documentCleans AS dc - JOIN documents AS d ON d.id = dc.documentId - LEFT JOIN collectors as c ON d.jobId = c.jobId - LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId - LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId - and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) - WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1) + SELECT 1 FROM currentCleanEntries WHERE documentId = $1 ) ` // IsDocumentClean // // SELECT EXISTS( -// SELECT 1 -// FROM documentCleans AS dc -// JOIN documents AS d ON d.id = dc.documentId -// LEFT JOIN collectors as c ON d.jobId = c.jobId -// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId -// LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId -// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) -// WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1) +// SELECT 1 FROM currentCleanEntries WHERE documentId = $1 // ) func (q *Queries) IsDocumentClean(ctx context.Context, documentid pgtype.UUID) (bool, error) { row := q.db.QueryRow(ctx, isDocumentClean, documentid) diff --git a/internal/database/repository/clean_test.go b/internal/database/repository/clean_test.go index 5609990a..315d0b72 100644 --- a/internal/database/repository/clean_test.go +++ b/internal/database/repository/clean_test.go @@ -9,6 +9,7 @@ import ( "queryorchestration/internal/test" "testing" + "github.com/jackc/pgx/v5/pgtype" "github.com/stretchr/testify/assert" ) @@ -65,10 +66,10 @@ func TestClean(t *testing.T) { clean, err := queries.GetDocumentCleanEntry(ctx, id) assert.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentCleanEntryRow{ - Documentid: id, - Bucket: bucket, - Key: key, - Version: 1, - }, clean) + assert.Equal(t, id, clean.Documentid) + assert.Equal(t, bucket, clean.Bucket) + assert.Equal(t, key, clean.Key) + assert.Equal(t, int32(1), clean.Version) + assert.NotEqual(t, pgtype.UUID{}, clean.ID) + assert.True(t, clean.ID.Valid) } diff --git a/internal/database/repository/client.sql.go b/internal/database/repository/client.sql.go index 13cc0dd8..86821e6a 100644 --- a/internal/database/repository/client.sql.go +++ b/internal/database/repository/client.sql.go @@ -43,15 +43,9 @@ func (q *Queries) CreateClient(ctx context.Context, name string) (pgtype.UUID, e } const getClient = `-- name: GetClient :one -SELECT c.id, c.name, coalesce(cs.canSync, false) as canSync +SELECT c.id, c.name, cs.canSync FROM clients as c - LEFT JOIN ( - SELECT clientId, canSync - FROM clientCanSync - WHERE clientId = $1 - ORDER BY id DESC - LIMIT 1 - ) as cs on cs.clientId = c.id + JOIN currentClientCanSync as cs on cs.clientId = c.id WHERE c.id = $1 ` @@ -63,18 +57,12 @@ type GetClientRow struct { // GetClient // -// SELECT c.id, c.name, coalesce(cs.canSync, false) as canSync +// SELECT c.id, c.name, cs.canSync // FROM clients as c -// LEFT JOIN ( -// SELECT clientId, canSync -// FROM clientCanSync -// WHERE clientId = $1 -// ORDER BY id DESC -// LIMIT 1 -// ) as cs on cs.clientId = c.id +// JOIN currentClientCanSync as cs on cs.clientId = c.id // WHERE c.id = $1 -func (q *Queries) GetClient(ctx context.Context, clientid pgtype.UUID) (*GetClientRow, error) { - row := q.db.QueryRow(ctx, getClient, clientid) +func (q *Queries) GetClient(ctx context.Context, id pgtype.UUID) (*GetClientRow, error) { + row := q.db.QueryRow(ctx, getClient, id) var i GetClientRow err := row.Scan(&i.ID, &i.Name, &i.Cansync) return &i, err diff --git a/internal/database/repository/collector.sql.go b/internal/database/repository/collector.sql.go index 3f608f1c..f70576a0 100644 --- a/internal/database/repository/collector.sql.go +++ b/internal/database/repository/collector.sql.go @@ -12,11 +12,11 @@ import ( ) const addCollectorQuery = `-- name: AddCollectorQuery :exec -INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4) +INSERT INTO collectorQueries (jobId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4) ` type AddCollectorQueryParams struct { - Collectorid pgtype.UUID `db:"collectorid"` + Jobid pgtype.UUID `db:"jobid"` Name string `db:"name"` Queryid pgtype.UUID `db:"queryid"` Addedversion int32 `db:"addedversion"` @@ -24,10 +24,10 @@ type AddCollectorQueryParams struct { // AddCollectorQuery // -// INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4) +// INSERT INTO collectorQueries (jobId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4) func (q *Queries) AddCollectorQuery(ctx context.Context, arg *AddCollectorQueryParams) error { _, err := q.db.Exec(ctx, addCollectorQuery, - arg.Collectorid, + arg.Jobid, arg.Name, arg.Queryid, arg.Addedversion, @@ -36,45 +36,30 @@ func (q *Queries) AddCollectorQuery(ctx context.Context, arg *AddCollectorQueryP } const addLatestCollectorVersion = `-- name: AddLatestCollectorVersion :one -INSERT INTO collectorVersions (collectorId) VALUES ($1) RETURNING id +INSERT INTO collectorVersions (jobId) VALUES ($1) RETURNING id ` // AddLatestCollectorVersion // -// INSERT INTO collectorVersions (collectorId) VALUES ($1) RETURNING id -func (q *Queries) AddLatestCollectorVersion(ctx context.Context, collectorid pgtype.UUID) (int32, error) { - row := q.db.QueryRow(ctx, addLatestCollectorVersion, collectorid) +// INSERT INTO collectorVersions (jobId) VALUES ($1) RETURNING id +func (q *Queries) AddLatestCollectorVersion(ctx context.Context, jobid pgtype.UUID) (int32, error) { + row := q.db.QueryRow(ctx, addLatestCollectorVersion, jobid) var id int32 err := row.Scan(&id) return id, err } -const createCollector = `-- name: CreateCollector :one -INSERT INTO collectors (jobId) VALUES ($1) RETURNING id -` - -// CreateCollector -// -// INSERT INTO collectors (jobId) VALUES ($1) RETURNING id -func (q *Queries) CreateCollector(ctx context.Context, jobid pgtype.UUID) (pgtype.UUID, error) { - row := q.db.QueryRow(ctx, createCollector, jobid) - var id pgtype.UUID - err := row.Scan(&id) - return id, err -} - const getCollector = `-- name: GetCollector :one -SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE id = $1 LIMIT 1 +SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1 ` // GetCollector // -// SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE id = $1 LIMIT 1 -func (q *Queries) GetCollector(ctx context.Context, id pgtype.UUID) (*Fullactivecollector, error) { - row := q.db.QueryRow(ctx, getCollector, id) +// SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1 +func (q *Queries) GetCollector(ctx context.Context, jobid pgtype.UUID) (*Fullactivecollector, error) { + row := q.db.QueryRow(ctx, getCollector, jobid) var i Fullactivecollector err := row.Scan( - &i.ID, &i.Jobid, &i.Mincleanversion, &i.Mintextversion, @@ -86,17 +71,16 @@ func (q *Queries) GetCollector(ctx context.Context, id pgtype.UUID) (*Fullactive } const getCollectorByJobID = `-- name: GetCollectorByJobID :one -SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1 +SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1 ` // GetCollectorByJobID // -// SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1 +// SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1 func (q *Queries) GetCollectorByJobID(ctx context.Context, jobid pgtype.UUID) (*Fullactivecollector, error) { row := q.db.QueryRow(ctx, getCollectorByJobID, jobid) var i Fullactivecollector err := row.Scan( - &i.ID, &i.Jobid, &i.Mincleanversion, &i.Mintextversion, @@ -108,14 +92,14 @@ func (q *Queries) GetCollectorByJobID(ctx context.Context, jobid pgtype.UUID) (* } const listCollectorQueries = `-- name: ListCollectorQueries :many -SELECT collectorid, jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1 +SELECT jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE jobId = $1 ` // ListCollectorQueries // -// SELECT collectorid, jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1 -func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.UUID) ([]*Collectorquerydependencytree, error) { - rows, err := q.db.Query(ctx, listCollectorQueries, collectorid) +// SELECT jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE jobId = $1 +func (q *Queries) ListCollectorQueries(ctx context.Context, jobid pgtype.UUID) ([]*Collectorquerydependencytree, error) { + rows, err := q.db.Query(ctx, listCollectorQueries, jobid) if err != nil { return nil, err } @@ -124,7 +108,6 @@ func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.U for rows.Next() { var i Collectorquerydependencytree if err := rows.Scan( - &i.Collectorid, &i.Jobid, &i.Queryid, &i.Type, @@ -142,72 +125,72 @@ func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.U } const removeCollectorQuery = `-- name: RemoveCollectorQuery :exec -UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and collectorId = $3 and removedVersion is null +UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and jobId = $3 and removedVersion is null ` type RemoveCollectorQueryParams struct { Removedversion *int32 `db:"removedversion"` Queryid pgtype.UUID `db:"queryid"` - Collectorid pgtype.UUID `db:"collectorid"` + Jobid pgtype.UUID `db:"jobid"` } // RemoveCollectorQuery // -// UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and collectorId = $3 and removedVersion is null +// UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and jobId = $3 and removedVersion is null func (q *Queries) RemoveCollectorQuery(ctx context.Context, arg *RemoveCollectorQueryParams) error { - _, err := q.db.Exec(ctx, removeCollectorQuery, arg.Removedversion, arg.Queryid, arg.Collectorid) + _, err := q.db.Exec(ctx, removeCollectorQuery, arg.Removedversion, arg.Queryid, arg.Jobid) return err } const setActiveCollectorVersion = `-- name: SetActiveCollectorVersion :exec -INSERT INTO collectorActiveVersions (collectorId, versionId) VALUES ($1, $2) +INSERT INTO collectorActiveVersions (jobId, versionId) VALUES ($1, $2) ` type SetActiveCollectorVersionParams struct { - Collectorid pgtype.UUID `db:"collectorid"` - Versionid int32 `db:"versionid"` + Jobid pgtype.UUID `db:"jobid"` + Versionid int32 `db:"versionid"` } // SetActiveCollectorVersion // -// INSERT INTO collectorActiveVersions (collectorId, versionId) VALUES ($1, $2) +// INSERT INTO collectorActiveVersions (jobId, versionId) VALUES ($1, $2) func (q *Queries) SetActiveCollectorVersion(ctx context.Context, arg *SetActiveCollectorVersionParams) error { - _, err := q.db.Exec(ctx, setActiveCollectorVersion, arg.Collectorid, arg.Versionid) + _, err := q.db.Exec(ctx, setActiveCollectorVersion, arg.Jobid, arg.Versionid) return err } const setCollectorCleanVersion = `-- name: SetCollectorCleanVersion :exec -INSERT INTO collectorMinCleanVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3) +INSERT INTO collectorMinCleanVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3) ` type SetCollectorCleanVersionParams struct { - Collectorid pgtype.UUID `db:"collectorid"` + Jobid pgtype.UUID `db:"jobid"` Addedversion int32 `db:"addedversion"` Versionid int32 `db:"versionid"` } // SetCollectorCleanVersion // -// INSERT INTO collectorMinCleanVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3) +// INSERT INTO collectorMinCleanVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3) func (q *Queries) SetCollectorCleanVersion(ctx context.Context, arg *SetCollectorCleanVersionParams) error { - _, err := q.db.Exec(ctx, setCollectorCleanVersion, arg.Collectorid, arg.Addedversion, arg.Versionid) + _, err := q.db.Exec(ctx, setCollectorCleanVersion, arg.Jobid, arg.Addedversion, arg.Versionid) return err } const setCollectorTextVersion = `-- name: SetCollectorTextVersion :exec -INSERT INTO collectorMinTextVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3) +INSERT INTO collectorMinTextVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3) ` type SetCollectorTextVersionParams struct { - Collectorid pgtype.UUID `db:"collectorid"` + Jobid pgtype.UUID `db:"jobid"` Addedversion int32 `db:"addedversion"` Versionid int32 `db:"versionid"` } // SetCollectorTextVersion // -// INSERT INTO collectorMinTextVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3) +// INSERT INTO collectorMinTextVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3) func (q *Queries) SetCollectorTextVersion(ctx context.Context, arg *SetCollectorTextVersionParams) error { - _, err := q.db.Exec(ctx, setCollectorTextVersion, arg.Collectorid, arg.Addedversion, arg.Versionid) + _, err := q.db.Exec(ctx, setCollectorTextVersion, arg.Jobid, arg.Addedversion, arg.Versionid) return err } diff --git a/internal/database/repository/collector_test.go b/internal/database/repository/collector_test.go index cf91cc8b..be2a0a14 100644 --- a/internal/database/repository/collector_test.go +++ b/internal/database/repository/collector_test.go @@ -38,11 +38,16 @@ func TestCollector(t *testing.T) { assert.NoError(t, err) version, err := queries.AddLatestQueryVersion(ctx, jsonId) assert.NoError(t, err) + err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ + Queryid: jsonId, + Versionid: version, + }) + assert.NoError(t, err) assert.Equal(t, int32(1), version) err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ Queryid: jsonId, Requiredqueryid: contextId, - Addedversion: 1, + Addedversion: version, }) assert.NoError(t, err) @@ -53,13 +58,9 @@ func TestCollector(t *testing.T) { minCleanVersion := int32(2) minTextVersion := int32(4) - collId, err := queries.CreateCollector(ctx, jobId) - assert.NoError(t, err) - - coll, err := queries.GetCollector(ctx, collId) + coll, err := queries.GetCollector(ctx, jobId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - ID: collId, Jobid: jobId, Mincleanversion: 0, Mintextversion: 0, @@ -70,7 +71,6 @@ func TestCollector(t *testing.T) { coll, err = queries.GetCollectorByJobID(ctx, jobId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - ID: collId, Jobid: jobId, Mincleanversion: 0, Mintextversion: 0, @@ -78,14 +78,13 @@ func TestCollector(t *testing.T) { Latestversion: 0, }, coll) - version, err = queries.AddLatestCollectorVersion(ctx, collId) + version, err = queries.AddLatestCollectorVersion(ctx, jobId) assert.NoError(t, err) assert.Equal(t, int32(1), version) - coll, err = queries.GetCollector(ctx, collId) + coll, err = queries.GetCollector(ctx, jobId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - ID: collId, Jobid: jobId, Mincleanversion: 0, Mintextversion: 0, @@ -94,15 +93,14 @@ func TestCollector(t *testing.T) { }, coll) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Versionid: 1, - Collectorid: collId, + Versionid: 1, + Jobid: jobId, }) assert.NoError(t, err) - coll, err = queries.GetCollector(ctx, collId) + coll, err = queries.GetCollector(ctx, jobId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - ID: collId, Jobid: jobId, Mincleanversion: 0, Mintextversion: 0, @@ -111,7 +109,7 @@ func TestCollector(t *testing.T) { }, coll) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Collectorid: collId, + Jobid: jobId, Queryid: jsonId, Addedversion: 1, Name: "example_key", @@ -119,16 +117,15 @@ func TestCollector(t *testing.T) { assert.NoError(t, err) err = queries.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{ - Collectorid: collId, + Jobid: jobId, Addedversion: 1, Versionid: minTextVersion, }) assert.NoError(t, err) - coll, err = queries.GetCollector(ctx, collId) + coll, err = queries.GetCollector(ctx, jobId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - ID: collId, Jobid: jobId, Mincleanversion: 0, Mintextversion: minTextVersion, @@ -138,16 +135,15 @@ func TestCollector(t *testing.T) { }, coll) err = queries.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{ - Collectorid: collId, + Jobid: jobId, Addedversion: 1, Versionid: minCleanVersion, }) assert.NoError(t, err) - coll, err = queries.GetCollector(ctx, collId) + coll, err = queries.GetCollector(ctx, jobId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - ID: collId, Jobid: jobId, Mincleanversion: minCleanVersion, Mintextversion: minTextVersion, @@ -156,20 +152,18 @@ func TestCollector(t *testing.T) { Fields: []byte(fmt.Sprintf("{\"example_key\": \"%s\"}", database.MustToUUID(jsonId).String())), }, coll) - qs, err := queries.ListCollectorQueries(ctx, collId) + qs, err := queries.ListCollectorQueries(ctx, jobId) assert.NoError(t, err) assert.Len(t, qs, 2) assert.ElementsMatch(t, []*repository.Collectorquerydependencytree{ { - Collectorid: collId, Jobid: jobId, Queryid: jsonId, - Queryversion: 0, + Queryversion: 1, Type: repository.QuerytypeJsonExtractor, Requiredids: []pgtype.UUID{contextId}, }, { - Collectorid: collId, Jobid: jobId, Queryid: contextId, Queryversion: 0, @@ -178,34 +172,33 @@ func TestCollector(t *testing.T) { }, }, qs) - version, err = queries.AddLatestCollectorVersion(ctx, collId) + version, err = queries.AddLatestCollectorVersion(ctx, jobId) assert.NoError(t, err) assert.Equal(t, int32(2), version) err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{ - Collectorid: collId, + Jobid: jobId, Queryid: jsonId, Removedversion: &version, }) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Versionid: 2, - Collectorid: collId, + Versionid: 2, + Jobid: jobId, }) assert.NoError(t, err) err = queries.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{ - Collectorid: collId, + Jobid: jobId, Versionid: minTextVersion + 1, Addedversion: version, }) assert.NoError(t, err) - coll, err = queries.GetCollector(ctx, collId) + coll, err = queries.GetCollector(ctx, jobId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - ID: collId, Jobid: jobId, Mincleanversion: minCleanVersion, Mintextversion: minTextVersion + 1, @@ -215,16 +208,15 @@ func TestCollector(t *testing.T) { }, coll) err = queries.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{ - Collectorid: collId, + Jobid: jobId, Versionid: minCleanVersion + 1, Addedversion: version, }) assert.NoError(t, err) - coll, err = queries.GetCollector(ctx, collId) + coll, err = queries.GetCollector(ctx, jobId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - ID: collId, Jobid: jobId, Mincleanversion: minCleanVersion + 1, Mintextversion: minTextVersion + 1, diff --git a/internal/database/repository/job.sql.go b/internal/database/repository/job.sql.go index b7f209b9..1eaada5d 100644 --- a/internal/database/repository/job.sql.go +++ b/internal/database/repository/job.sql.go @@ -43,15 +43,9 @@ func (q *Queries) CreateJob(ctx context.Context, clientid pgtype.UUID) (pgtype.U } const getJob = `-- name: GetJob :one -SELECT j.id, j.clientId, coalesce(cs.canSync, false) as canSync +SELECT j.id, j.clientId, cs.canSync FROM jobs as j - LEFT JOIN ( - SELECT jobId, canSync - FROM jobCanSync - WHERE jobId = $1 - ORDER BY id DESC - LIMIT 1 - ) as cs on cs.jobId = j.id + JOIN currentJobCanSync as cs on cs.jobId = j.id WHERE j.id = $1 ` @@ -63,23 +57,177 @@ type GetJobRow struct { // GetJob // -// SELECT j.id, j.clientId, coalesce(cs.canSync, false) as canSync +// SELECT j.id, j.clientId, cs.canSync // FROM jobs as j -// LEFT JOIN ( -// SELECT jobId, canSync -// FROM jobCanSync -// WHERE jobId = $1 -// ORDER BY id DESC -// LIMIT 1 -// ) as cs on cs.jobId = j.id +// JOIN currentJobCanSync as cs on cs.jobId = j.id // WHERE j.id = $1 -func (q *Queries) GetJob(ctx context.Context, jobid pgtype.UUID) (*GetJobRow, error) { - row := q.db.QueryRow(ctx, getJob, jobid) +func (q *Queries) GetJob(ctx context.Context, id pgtype.UUID) (*GetJobRow, error) { + row := q.db.QueryRow(ctx, getJob, id) var i GetJobRow err := row.Scan(&i.ID, &i.Clientid, &i.Cansync) return &i, err } +const isJobSynced = `-- name: IsJobSynced :one +WITH +docs AS ( + -- Get all documents for this job + SELECT id, jobId FROM documents WHERE jobId = $1 +), +doc_text_entries AS ( + -- Documents with their current text entries + SELECT + d.id AS document_id, + cte.id AS text_entry_id + FROM + docs d + LEFT JOIN currentTextEntries cte ON cte.documentId = d.id +), +required_results AS ( + -- All required document-query-version combinations + SELECT + d.id AS document_id, + cqdt.queryId, + cqdt.queryVersion, + dte.text_entry_id + FROM + docs d + JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.jobId + JOIN doc_text_entries dte ON dte.document_id = d.id + and dte.text_entry_id IS NOT NULL +), +existing_results AS ( + -- Valid results that exist + SELECT + rr.document_id AS document_id, + r.queryId, + r.id AS result_id + FROM + results r + JOIN required_results rr ON + r.queryId = rr.queryId AND + r.queryVersion = rr.queryVersion AND + r.textEntryId = rr.text_entry_id +), +missing_results AS ( + -- Find missing results + SELECT rr.queryId + FROM required_results rr + LEFT JOIN existing_results er on er.queryId = rr.queryId + WHERE er.result_id is null +), +dependency_check AS ( + -- Check for missing dependencies + SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId + FROM existing_results er + JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId + LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id + WHERE rd.resultId is null +) +SELECT ( + -- No documents means job is synced + NOT EXISTS (SELECT 1 FROM docs) + + OR + + -- Documents with no text entries + (NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL) + + and + + -- Documents with missing results + NOT EXISTS (SELECT 1 FROM missing_results) + + and + + -- Documents with missing dependencies + NOT EXISTS (SELECT 1 FROM dependency_check)) +)::bool as is_synced +` + +// IsJobSynced +// +// WITH +// docs AS ( +// -- Get all documents for this job +// SELECT id, jobId FROM documents WHERE jobId = $1 +// ), +// doc_text_entries AS ( +// -- Documents with their current text entries +// SELECT +// d.id AS document_id, +// cte.id AS text_entry_id +// FROM +// docs d +// LEFT JOIN currentTextEntries cte ON cte.documentId = d.id +// ), +// required_results AS ( +// -- All required document-query-version combinations +// SELECT +// d.id AS document_id, +// cqdt.queryId, +// cqdt.queryVersion, +// dte.text_entry_id +// FROM +// docs d +// JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.jobId +// JOIN doc_text_entries dte ON dte.document_id = d.id +// and dte.text_entry_id IS NOT NULL +// ), +// existing_results AS ( +// -- Valid results that exist +// SELECT +// rr.document_id AS document_id, +// r.queryId, +// r.id AS result_id +// FROM +// results r +// JOIN required_results rr ON +// r.queryId = rr.queryId AND +// r.queryVersion = rr.queryVersion AND +// r.textEntryId = rr.text_entry_id +// ), +// missing_results AS ( +// -- Find missing results +// SELECT rr.queryId +// FROM required_results rr +// LEFT JOIN existing_results er on er.queryId = rr.queryId +// WHERE er.result_id is null +// ), +// dependency_check AS ( +// -- Check for missing dependencies +// SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId +// FROM existing_results er +// JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId +// LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id +// WHERE rd.resultId is null +// ) +// SELECT ( +// -- No documents means job is synced +// NOT EXISTS (SELECT 1 FROM docs) +// +// OR +// +// -- Documents with no text entries +// (NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL) +// +// and +// +// -- Documents with missing results +// NOT EXISTS (SELECT 1 FROM missing_results) +// +// and +// +// -- Documents with missing dependencies +// NOT EXISTS (SELECT 1 FROM dependency_check)) +// )::bool as is_synced +func (q *Queries) IsJobSynced(ctx context.Context, dollar_1 pgtype.UUID) (bool, error) { + row := q.db.QueryRow(ctx, isJobSynced, dollar_1) + var is_synced bool + err := row.Scan(&is_synced) + return is_synced, err +} + const listJobDocumentIDsBatch = `-- name: ListJobDocumentIDsBatch :many SELECT id, totalCount FROM listJobDocumentIDs($1, $2, $3) ` diff --git a/internal/database/repository/job_test.go b/internal/database/repository/job_test.go index b510bf25..2fb5dd3f 100644 --- a/internal/database/repository/job_test.go +++ b/internal/database/repository/job_test.go @@ -164,3 +164,428 @@ func TestListJobDocumentIDs(t *testing.T) { }, }, ids) } + +func TestJobSync(t *testing.T) { + if testing.Short() { + t.Skip("Skipping long test in short mode") + } + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + test.SetCfgProvider(t, cfg) + cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../..")) + _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Cfg: cfg, + RunMigrations: true, + }) + defer cleanup() + + queries := cfg.GetDBQueries() + + contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) + assert.NoError(t, err) + _, err = queries.AddLatestQueryVersion(ctx, contextQueryID) + assert.NoError(t, err) + err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ + Queryid: contextQueryID, + Versionid: 1, + }) + assert.NoError(t, err) + jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) + assert.NoError(t, err) + _, err = queries.AddLatestQueryVersion(ctx, jsonQueryID) + assert.NoError(t, err) + err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ + Queryid: jsonQueryID, + Versionid: 1, + }) + assert.NoError(t, err) + err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ + Queryid: jsonQueryID, + Requiredqueryid: contextQueryID, + Addedversion: 1, + }) + assert.NoError(t, err) + + clientId, err := queries.CreateClient(ctx, "example_client") + assert.NoError(t, err) + jobId, err := queries.CreateJob(ctx, clientId) + assert.NoError(t, err) + + _, err = queries.AddLatestCollectorVersion(ctx, jobId) + assert.NoError(t, err) + err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ + Versionid: 1, + Jobid: jobId, + }) + assert.NoError(t, err) + err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + Jobid: jobId, + Queryid: jsonQueryID, + Addedversion: 1, + Name: "example_key", + }) + assert.NoError(t, err) + + isSynced, err := queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, isSynced) + + documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + Jobid: jobId, + Hash: "example_hash", + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ + Documentid: documentID, + Version: 1, + Bucket: "hi", + Key: "hello", + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID) + assert.NoError(t, err) + + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Bucket: "hi", + Key: "hello", + Cleanentryid: cleanentry.ID, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + textentry, err := queries.GetDocumentTextEntry(ctx, documentID) + assert.NoError(t, err) + + depresultid, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "example_value", + Textentryid: textentry.ID, + Queryversion: 1, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + resultid, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "example_value", + Textentryid: textentry.ID, + Queryversion: 1, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: resultid, + Requiredresultid: depresultid, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, isSynced) + + _, err = queries.AddLatestQueryVersion(ctx, contextQueryID) + assert.NoError(t, err) + err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ + Queryid: contextQueryID, + Versionid: 2, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "example_value", + Textentryid: textentry.ID, + Queryversion: 2, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + resultid, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "example_value", + Textentryid: textentry.ID, + Queryversion: 1, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: resultid, + Requiredresultid: depresultid, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, isSynced) + + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Bucket: "hi", + Key: "hello", + Cleanentryid: cleanentry.ID, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + textentry, err = queries.GetDocumentTextEntry(ctx, documentID) + assert.NoError(t, err) + + depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "example_value", + Textentryid: textentry.ID, + Queryversion: 2, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + resultid, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "example_value", + Textentryid: textentry.ID, + Queryversion: 1, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: resultid, + Requiredresultid: depresultid, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, isSynced) + + err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ + Documentid: documentID, + Version: 1, + Bucket: "hi", + Key: "hello", + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + cleanentry, err = queries.GetDocumentCleanEntry(ctx, documentID) + assert.NoError(t, err) + + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Bucket: "hi", + Key: "hello", + Cleanentryid: cleanentry.ID, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + textentry, err = queries.GetDocumentTextEntry(ctx, documentID) + assert.NoError(t, err) + + depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "example_value", + Textentryid: textentry.ID, + Queryversion: 2, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + resultid, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "example_value", + Textentryid: textentry.ID, + Queryversion: 1, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: resultid, + Requiredresultid: depresultid, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, isSynced) + + _, err = queries.AddLatestCollectorVersion(ctx, jobId) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, isSynced) + + collversion := int32(2) + err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ + Versionid: 2, + Jobid: jobId, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, isSynced) + + err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{ + Jobid: jobId, + Queryid: jsonQueryID, + Removedversion: &collversion, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, isSynced) + + err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + Jobid: jobId, + Queryid: jsonQueryID, + Addedversion: 2, + Name: "second_key", + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, isSynced) + + err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + Jobid: jobId, + Queryid: contextQueryID, + Addedversion: 2, + Name: "example_key", + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, isSynced) + + superQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) + assert.NoError(t, err) + _, err = queries.AddLatestQueryVersion(ctx, superQueryID) + assert.NoError(t, err) + err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ + Queryid: superQueryID, + Versionid: 1, + }) + assert.NoError(t, err) + err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ + Queryid: contextQueryID, + Requiredqueryid: superQueryID, + Addedversion: 1, + }) + assert.NoError(t, err) + + superresultid, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: superQueryID, + Value: "example_value", + Textentryid: textentry.ID, + Queryversion: 1, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "example_value", + Textentryid: textentry.ID, + Queryversion: 2, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + resultid, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "example_value", + Textentryid: textentry.ID, + Queryversion: 1, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: resultid, + Requiredresultid: depresultid, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, isSynced) + + err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: depresultid, + Requiredresultid: superresultid, + }) + assert.NoError(t, err) + + isSynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, isSynced) +} diff --git a/internal/database/repository/models.go b/internal/database/repository/models.go index d8998991..fc503074 100644 --- a/internal/database/repository/models.go +++ b/internal/database/repository/models.go @@ -73,30 +73,25 @@ type Clientcansync struct { Cansync bool `db:"cansync"` } -type Collector struct { - ID pgtype.UUID `db:"id"` - Jobid pgtype.UUID `db:"jobid"` -} - type Collectoractiveversion struct { - ID pgtype.UUID `db:"id"` - Collectorid pgtype.UUID `db:"collectorid"` - Versionid int32 `db:"versionid"` + ID pgtype.UUID `db:"id"` + Jobid pgtype.UUID `db:"jobid"` + Versionid int32 `db:"versionid"` } type Collectorcurrentactiveversion struct { - Collectorid pgtype.UUID `db:"collectorid"` - ID int32 `db:"id"` + Jobid pgtype.UUID `db:"jobid"` + Activeversion int32 `db:"activeversion"` } type Collectorlatestversion struct { - Collectorid pgtype.UUID `db:"collectorid"` - ID int32 `db:"id"` + Jobid pgtype.UUID `db:"jobid"` + Latestversion int32 `db:"latestversion"` } type Collectormincleanversion struct { ID pgtype.UUID `db:"id"` - Collectorid pgtype.UUID `db:"collectorid"` + Jobid pgtype.UUID `db:"jobid"` Versionid int32 `db:"versionid"` Addedversion int32 `db:"addedversion"` Removedversion *int32 `db:"removedversion"` @@ -104,7 +99,7 @@ type Collectormincleanversion struct { type Collectormintextversion struct { ID pgtype.UUID `db:"id"` - Collectorid pgtype.UUID `db:"collectorid"` + Jobid pgtype.UUID `db:"jobid"` Versionid int32 `db:"versionid"` Addedversion int32 `db:"addedversion"` Removedversion *int32 `db:"removedversion"` @@ -112,7 +107,7 @@ type Collectormintextversion struct { type Collectorquery struct { ID pgtype.UUID `db:"id"` - Collectorid pgtype.UUID `db:"collectorid"` + Jobid pgtype.UUID `db:"jobid"` Name string `db:"name"` Queryid pgtype.UUID `db:"queryid"` Addedversion int32 `db:"addedversion"` @@ -120,7 +115,6 @@ type Collectorquery struct { } type Collectorquerydependencytree struct { - Collectorid pgtype.UUID `db:"collectorid"` Jobid pgtype.UUID `db:"jobid"` Queryid pgtype.UUID `db:"queryid"` Type Querytype `db:"type"` @@ -129,9 +123,57 @@ type Collectorquerydependencytree struct { } type Collectorversion struct { - Collectorid pgtype.UUID `db:"collectorid"` - ID int32 `db:"id"` - Addedat pgtype.Timestamp `db:"addedat"` + Jobid pgtype.UUID `db:"jobid"` + ID int32 `db:"id"` + Addedat pgtype.Timestamp `db:"addedat"` +} + +type Currentcleanentry struct { + ID pgtype.UUID `db:"id"` + Documentid pgtype.UUID `db:"documentid"` + Bucket string `db:"bucket"` + Key string `db:"key"` + Version int32 `db:"version"` +} + +type Currentclientcansync struct { + Clientid pgtype.UUID `db:"clientid"` + Cansync bool `db:"cansync"` +} + +type Currentcollectormincleanversion struct { + Jobid pgtype.UUID `db:"jobid"` + Mincleanversion int32 `db:"mincleanversion"` +} + +type Currentcollectormintextversion struct { + Jobid pgtype.UUID `db:"jobid"` + Mintextversion int32 `db:"mintextversion"` +} + +type Currentcollectorqueriesjsonagg struct { + Jobid pgtype.UUID `db:"jobid"` + Fields []byte `db:"fields"` +} + +type Currentcollectorquery struct { + Jobid pgtype.UUID `db:"jobid"` + Name *string `db:"name"` + Queryid pgtype.UUID `db:"queryid"` +} + +type Currentjobcansync struct { + Jobid pgtype.UUID `db:"jobid"` + Cansync bool `db:"cansync"` +} + +type Currenttextentry struct { + ID pgtype.UUID `db:"id"` + Documentid pgtype.UUID `db:"documentid"` + Bucket string `db:"bucket"` + Key string `db:"key"` + Version int32 `db:"version"` + Cleanentryid pgtype.UUID `db:"cleanentryid"` } type Document struct { @@ -156,15 +198,14 @@ type Documententry struct { } type Documenttextextraction struct { - ID pgtype.UUID `db:"id"` - Documentid pgtype.UUID `db:"documentid"` - Version int32 `db:"version"` - Bucket string `db:"bucket"` - Key string `db:"key"` + ID pgtype.UUID `db:"id"` + Cleanentryid pgtype.UUID `db:"cleanentryid"` + Version int32 `db:"version"` + Bucket string `db:"bucket"` + Key string `db:"key"` } type Fullactivecollector struct { - ID pgtype.UUID `db:"id"` Jobid pgtype.UUID `db:"jobid"` Mincleanversion int32 `db:"mincleanversion"` Mintextversion int32 `db:"mintextversion"` @@ -218,13 +259,28 @@ type Queryconfig struct { } type Querycurrentactiveversion struct { + Queryid pgtype.UUID `db:"queryid"` + Activeversion int32 `db:"activeversion"` +} + +type Querycurrentconfig struct { Queryid pgtype.UUID `db:"queryid"` - ID int32 `db:"id"` + Config []byte `db:"config"` +} + +type Querycurrentrequiredid struct { + Queryid pgtype.UUID `db:"queryid"` + Requiredqueryid pgtype.UUID `db:"requiredqueryid"` +} + +type Querycurrentrequiredidsagg struct { + Queryid pgtype.UUID `db:"queryid"` + Requiredids []pgtype.UUID `db:"requiredids"` } type Querylatestversion struct { - Queryid pgtype.UUID `db:"queryid"` - ID int32 `db:"id"` + Queryid pgtype.UUID `db:"queryid"` + Latestversion int32 `db:"latestversion"` } type Queryversion struct { @@ -243,10 +299,13 @@ type Requiredquery struct { type Result struct { ID pgtype.UUID `db:"id"` + Textentryid pgtype.UUID `db:"textentryid"` Queryid pgtype.UUID `db:"queryid"` - Documentid pgtype.UUID `db:"documentid"` Value string `db:"value"` - Cleanversion int32 `db:"cleanversion"` - Textversion int32 `db:"textversion"` Queryversion int32 `db:"queryversion"` } + +type Resultdependency struct { + Resultid pgtype.UUID `db:"resultid"` + Requiredresultid pgtype.UUID `db:"requiredresultid"` +} diff --git a/internal/database/repository/query.sql.go b/internal/database/repository/query.sql.go index c1281f7f..c8285988 100644 --- a/internal/database/repository/query.sql.go +++ b/internal/database/repository/query.sql.go @@ -92,6 +92,20 @@ func (q *Queries) CreateQuery(ctx context.Context, type_ Querytype) (pgtype.UUID return id, err } +const getActiveQueryConfig = `-- name: GetActiveQueryConfig :one +SELECT config FROM queryCurrentConfigs where queryId = $1 +` + +// GetActiveQueryConfig +// +// SELECT config FROM queryCurrentConfigs where queryId = $1 +func (q *Queries) GetActiveQueryConfig(ctx context.Context, queryid pgtype.UUID) ([]byte, error) { + row := q.db.QueryRow(ctx, getActiveQueryConfig, queryid) + var config []byte + err := row.Scan(&config) + return config, err +} + const getQuery = `-- name: GetQuery :one SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = $1 ` @@ -113,30 +127,6 @@ func (q *Queries) GetQuery(ctx context.Context, id pgtype.UUID) (*Fullactivequer return &i, err } -const getQueryConfig = `-- name: GetQueryConfig :one -SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 and COALESCE(removedVersion, $2 - 1) < $2 -` - -type GetQueryConfigParams struct { - Queryid pgtype.UUID `db:"queryid"` - Addedversion int32 `db:"addedversion"` -} - -type GetQueryConfigRow struct { - ID pgtype.UUID `db:"id"` - Config []byte `db:"config"` -} - -// GetQueryConfig -// -// SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 and COALESCE(removedVersion, $2 - 1) < $2 -func (q *Queries) GetQueryConfig(ctx context.Context, arg *GetQueryConfigParams) (*GetQueryConfigRow, error) { - row := q.db.QueryRow(ctx, getQueryConfig, arg.Queryid, arg.Addedversion) - var i GetQueryConfigRow - err := row.Scan(&i.ID, &i.Config) - return &i, err -} - const getQueryWithVersion = `-- name: GetQueryWithVersion :one WITH query as ( SELECT id, type FROM queries WHERE id = $1 @@ -149,22 +139,23 @@ SELECT c.queryId, c.config ), requiredIds as ( SELECT r.queryId, + coalesce( ARRAY_AGG(DISTINCT r.requiredQueryId) - FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[] - as requiredIds + FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[], + array[]::uuid[] + )::uuid[] as requiredIds FROM query AS q LEFT JOIN requiredQueries AS r ON q.id = r.queryId and isInVersion($2, r.addedVersion, r.removedVersion) GROUP BY r.queryId ) -SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config, - coalesce( - r.requiredIds, - array[]::uuid[] - )::uuid[] as requiredIds +SELECT DISTINCT q.id, q.type, + av.activeVersion, + lv.latestVersion, c.config, + r.requiredIds FROM query AS q - LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId - LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id + JOIN queryCurrentActiveVersions as av on q.id = av.queryId + JOIN queryLatestVersions as lv on lv.queryId = q.id LEFT JOIN config AS c ON q.id = c.queryId LEFT JOIN requiredIds AS r ON q.id = r.queryId ` @@ -196,22 +187,23 @@ type GetQueryWithVersionRow struct { // ), // requiredIds as ( // SELECT r.queryId, +// coalesce( // ARRAY_AGG(DISTINCT r.requiredQueryId) -// FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[] -// as requiredIds +// FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[], +// array[]::uuid[] +// )::uuid[] as requiredIds // FROM query AS q // LEFT JOIN requiredQueries AS r ON q.id = r.queryId // and isInVersion($2, r.addedVersion, r.removedVersion) // GROUP BY r.queryId // ) -// SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config, -// coalesce( -// r.requiredIds, -// array[]::uuid[] -// )::uuid[] as requiredIds +// SELECT DISTINCT q.id, q.type, +// av.activeVersion, +// lv.latestVersion, c.config, +// r.requiredIds // FROM query AS q -// LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId -// LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id +// JOIN queryCurrentActiveVersions as av on q.id = av.queryId +// JOIN queryLatestVersions as lv on lv.queryId = q.id // LEFT JOIN config AS c ON q.id = c.queryId // LEFT JOIN requiredIds AS r ON q.id = r.queryId func (q *Queries) GetQueryWithVersion(ctx context.Context, arg *GetQueryWithVersionParams) (*GetQueryWithVersionRow, error) { @@ -230,22 +222,28 @@ func (q *Queries) GetQueryWithVersion(ctx context.Context, arg *GetQueryWithVers const isQueryInDependencyTree = `-- name: IsQueryInDependencyTree :one SELECT EXISTS ( - SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1) + SELECT 1 FROM queryActiveDependencies + WHERE id = any($1) + and requiredQueryId = $2 + or $2 = any($1) ) ` type IsQueryInDependencyTreeParams struct { - ID []pgtype.UUID `db:"id"` - Requiredqueryid pgtype.UUID `db:"requiredqueryid"` + Requiredqueryids []pgtype.UUID `db:"requiredqueryids"` + Queryid pgtype.UUID `db:"queryid"` } // IsQueryInDependencyTree // // SELECT EXISTS ( -// SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1) +// SELECT 1 FROM queryActiveDependencies +// WHERE id = any($1) +// and requiredQueryId = $2 +// or $2 = any($1) // ) func (q *Queries) IsQueryInDependencyTree(ctx context.Context, arg *IsQueryInDependencyTreeParams) (bool, error) { - row := q.db.QueryRow(ctx, isQueryInDependencyTree, arg.ID, arg.Requiredqueryid) + row := q.db.QueryRow(ctx, isQueryInDependencyTree, arg.Requiredqueryids, arg.Queryid) var exists bool err := row.Scan(&exists) return exists, err @@ -324,14 +322,15 @@ WITH doc AS ( SELECT id, jobId FROM documents where id = $2 ) SELECT dt.queryId - FROM collectorQueryDependencyTree as dt - JOIN doc as d on d.jobId = dt.jobId - where $1 = any(dt.requiredIds) + FROM doc as d + JOIN collectorQueryDependencyTree as dt + on d.jobId = dt.jobId + and $1 = any(dt.requiredIds) ` type ListQueryDirectDependentsByDocumentIDParams struct { - Requiredids pgtype.UUID `db:"requiredids"` - ID pgtype.UUID `db:"id"` + Queryid pgtype.UUID `db:"queryid"` + Documentid pgtype.UUID `db:"documentid"` } // ListQueryDirectDependentsByDocumentID @@ -340,11 +339,12 @@ type ListQueryDirectDependentsByDocumentIDParams struct { // SELECT id, jobId FROM documents where id = $2 // ) // SELECT dt.queryId -// FROM collectorQueryDependencyTree as dt -// JOIN doc as d on d.jobId = dt.jobId -// where $1 = any(dt.requiredIds) +// FROM doc as d +// JOIN collectorQueryDependencyTree as dt +// on d.jobId = dt.jobId +// and $1 = any(dt.requiredIds) func (q *Queries) ListQueryDirectDependentsByDocumentID(ctx context.Context, arg *ListQueryDirectDependentsByDocumentIDParams) ([]pgtype.UUID, error) { - rows, err := q.db.Query(ctx, listQueryDirectDependentsByDocumentID, arg.Requiredids, arg.ID) + rows, err := q.db.Query(ctx, listQueryDirectDependentsByDocumentID, arg.Queryid, arg.Documentid) if err != nil { return nil, err } diff --git a/internal/database/repository/query_test.go b/internal/database/repository/query_test.go index 532bf53a..b02e7288 100644 --- a/internal/database/repository/query_test.go +++ b/internal/database/repository/query_test.go @@ -152,13 +152,6 @@ func TestQueries(t *testing.T) { }) assert.NoError(t, err) - err = queries.SetQueryConfig(ctx, &repository.SetQueryConfigParams{ - Queryid: jsonQueryID, - Config: []byte(`{"second":"key"}`), - Addedversion: 2, - }) - assert.NoError(t, err) - err = queries.SetQueryConfig(ctx, &repository.SetQueryConfigParams{ Queryid: jsonQueryID, Config: jsonConfig, @@ -166,29 +159,40 @@ func TestQueries(t *testing.T) { }) assert.NoError(t, err) - jsonQueryConfig, err := queries.GetQueryConfig(ctx, &repository.GetQueryConfigParams{ - Queryid: jsonQueryID, - Addedversion: jsonQuery.Activeversion, - }) + jsonQueryConfig, err := queries.GetActiveQueryConfig(ctx, jsonQueryID) assert.NoError(t, err) - assert.Equal(t, jsonConfig, jsonQueryConfig.Config) + assert.Equal(t, jsonConfig, jsonQueryConfig) - jsonQueryConfig, err = queries.GetQueryConfig(ctx, &repository.GetQueryConfigParams{ + err = queries.SetQueryConfig(ctx, &repository.SetQueryConfigParams{ Queryid: jsonQueryID, + Config: []byte(`{"second":"key"}`), Addedversion: 2, }) assert.NoError(t, err) - assert.Equal(t, []byte(`{"second": "key"}`), jsonQueryConfig.Config) + + jsonQueryConfig, err = queries.GetActiveQueryConfig(ctx, jsonQueryID) + assert.NoError(t, err) + assert.Equal(t, jsonConfig, jsonQueryConfig) + + err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ + Queryid: jsonQueryID, + Versionid: 2, + }) + assert.NoError(t, err) + + jsonQueryConfig, err = queries.GetActiveQueryConfig(ctx, jsonQueryID) + assert.NoError(t, err) + assert.Equal(t, []byte(`{"second": "key"}`), jsonQueryConfig) jsonQuery, err = queries.GetQuery(ctx, jsonQueryID) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivequery{ ID: jsonQueryID, Type: repository.QuerytypeJsonExtractor, - Activeversion: 1, + Activeversion: 2, Latestversion: 2, - Config: jsonConfig, - Requiredids: []pgtype.UUID{contextQueryID}, + Config: jsonQueryConfig, + Requiredids: []pgtype.UUID{}, }, jsonQuery) v := int32(1) @@ -200,7 +204,7 @@ func TestQueries(t *testing.T) { assert.EqualExportedValues(t, &repository.GetQueryWithVersionRow{ ID: jsonQueryID, Type: repository.QuerytypeJsonExtractor, - Activeversion: 1, + Activeversion: 2, Latestversion: 2, Config: jsonConfig, Requiredids: []pgtype.UUID{contextQueryID}, @@ -248,13 +252,11 @@ func TestQueryDependencyTree(t *testing.T) { assert.NoError(t, err) jobID, err := queries.CreateJob(ctx, clientID) assert.NoError(t, err) - collID, err := queries.CreateCollector(ctx, jobID) - assert.NoError(t, err) - _, err = queries.AddLatestCollectorVersion(ctx, collID) + version, err := queries.AddLatestCollectorVersion(ctx, jobID) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Collectorid: collID, - Versionid: 1, + Jobid: jobID, + Versionid: version, }) assert.NoError(t, err) docID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ @@ -265,10 +267,17 @@ func TestQueryDependencyTree(t *testing.T) { contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) assert.NoError(t, err) + _, err = queries.AddLatestQueryVersion(ctx, contextQueryID) + assert.NoError(t, err) + err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ + Queryid: contextQueryID, + Versionid: 1, + }) + assert.NoError(t, err) dependents, err := queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: contextQueryID, + Documentid: docID, + Queryid: contextQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{}, dependents) @@ -277,10 +286,15 @@ func TestQueryDependencyTree(t *testing.T) { assert.NoError(t, err) _, err = queries.AddLatestQueryVersion(ctx, jsonQueryID) assert.NoError(t, err) + err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ + Queryid: jsonQueryID, + Versionid: 1, + }) + assert.NoError(t, err) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: jsonQueryID, + Documentid: docID, + Queryid: jsonQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{}, dependents) @@ -293,20 +307,20 @@ func TestQueryDependencyTree(t *testing.T) { assert.NoError(t, err) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: jsonQueryID, + Documentid: docID, + Queryid: jsonQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: contextQueryID, + Documentid: docID, + Queryid: contextQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{}, dependents) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Collectorid: collID, + Jobid: jobID, Name: "example_name'", Queryid: jsonQueryID, Addedversion: 1, @@ -314,14 +328,14 @@ func TestQueryDependencyTree(t *testing.T) { assert.NoError(t, err) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: jsonQueryID, + Documentid: docID, + Queryid: jsonQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: contextQueryID, + Documentid: docID, + Queryid: contextQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents) @@ -330,16 +344,21 @@ func TestQueryDependencyTree(t *testing.T) { assert.NoError(t, err) _, err = queries.AddLatestQueryVersion(ctx, secondJsonQueryID) assert.NoError(t, err) + err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ + Queryid: secondJsonQueryID, + Versionid: 1, + }) + assert.NoError(t, err) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: jsonQueryID, + Documentid: docID, + Queryid: jsonQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: contextQueryID, + Documentid: docID, + Queryid: contextQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents) @@ -352,26 +371,26 @@ func TestQueryDependencyTree(t *testing.T) { assert.NoError(t, err) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: secondJsonQueryID, + Documentid: docID, + Queryid: secondJsonQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: jsonQueryID, + Documentid: docID, + Queryid: jsonQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: contextQueryID, + Documentid: docID, + Queryid: contextQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Collectorid: collID, + Jobid: jobID, Name: "example_name'", Queryid: secondJsonQueryID, Addedversion: 1, @@ -379,55 +398,55 @@ func TestQueryDependencyTree(t *testing.T) { assert.NoError(t, err) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: secondJsonQueryID, + Documentid: docID, + Queryid: secondJsonQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: jsonQueryID, + Documentid: docID, + Queryid: jsonQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{secondJsonQueryID}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: docID, - Requiredids: contextQueryID, + Documentid: docID, + Queryid: contextQueryID, }) assert.NoError(t, err) assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents) isdependent, err := queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{ - Requiredqueryid: jsonQueryID, - ID: []pgtype.UUID{contextQueryID}, + Queryid: jsonQueryID, + Requiredqueryids: []pgtype.UUID{contextQueryID}, }) assert.NoError(t, err) assert.False(t, isdependent) isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{ - Requiredqueryid: jsonQueryID, - ID: []pgtype.UUID{secondJsonQueryID}, + Queryid: jsonQueryID, + Requiredqueryids: []pgtype.UUID{secondJsonQueryID}, }) assert.NoError(t, err) assert.True(t, isdependent) isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{ - Requiredqueryid: jsonQueryID, - ID: []pgtype.UUID{jsonQueryID}, + Queryid: jsonQueryID, + Requiredqueryids: []pgtype.UUID{jsonQueryID}, }) assert.NoError(t, err) assert.True(t, isdependent) isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{ - Requiredqueryid: secondJsonQueryID, - ID: []pgtype.UUID{jsonQueryID, contextQueryID}, + Queryid: secondJsonQueryID, + Requiredqueryids: []pgtype.UUID{jsonQueryID, contextQueryID}, }) assert.NoError(t, err) assert.False(t, isdependent) isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{ - Requiredqueryid: contextQueryID, - ID: []pgtype.UUID{jsonQueryID, secondJsonQueryID}, + Queryid: contextQueryID, + Requiredqueryids: []pgtype.UUID{jsonQueryID, secondJsonQueryID}, }) assert.NoError(t, err) assert.True(t, isdependent) @@ -521,17 +540,15 @@ func TestListQueryJobs(t *testing.T) { assert.NoError(t, err) jobOneID, err := queries.CreateJob(ctx, clientOneID) assert.NoError(t, err) - collOneID, err := queries.CreateCollector(ctx, jobOneID) - assert.NoError(t, err) - _, err = queries.AddLatestCollectorVersion(ctx, collOneID) + versionOne, err := queries.AddLatestCollectorVersion(ctx, jobOneID) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Collectorid: collOneID, - Versionid: 1, + Jobid: jobOneID, + Versionid: versionOne, }) assert.NoError(t, err) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Collectorid: collOneID, + Jobid: jobOneID, Queryid: contextID, Addedversion: 1, Name: "example_key", @@ -544,17 +561,15 @@ func TestListQueryJobs(t *testing.T) { jobTwoID, err := queries.CreateJob(ctx, clientOneID) assert.NoError(t, err) - collTwoID, err := queries.CreateCollector(ctx, jobTwoID) - assert.NoError(t, err) - _, err = queries.AddLatestCollectorVersion(ctx, collTwoID) + versionTwo, err := queries.AddLatestCollectorVersion(ctx, jobTwoID) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Collectorid: collTwoID, - Versionid: 1, + Jobid: jobTwoID, + Versionid: versionTwo, }) assert.NoError(t, err) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Collectorid: collTwoID, + Jobid: jobTwoID, Queryid: contextID, Addedversion: 1, Name: "example_key", @@ -569,17 +584,15 @@ func TestListQueryJobs(t *testing.T) { assert.NoError(t, err) jobThreeID, err := queries.CreateJob(ctx, clientTwoID) assert.NoError(t, err) - collThreeID, err := queries.CreateCollector(ctx, jobThreeID) - assert.NoError(t, err) - _, err = queries.AddLatestCollectorVersion(ctx, collThreeID) + versionThree, err := queries.AddLatestCollectorVersion(ctx, jobThreeID) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Collectorid: collThreeID, - Versionid: 1, + Jobid: jobThreeID, + Versionid: versionThree, }) assert.NoError(t, err) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Collectorid: collThreeID, + Jobid: jobThreeID, Queryid: contextID, Addedversion: 1, Name: "example_key", @@ -601,7 +614,7 @@ func TestListQueryJobs(t *testing.T) { }) assert.NoError(t, err) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Collectorid: collOneID, + Jobid: jobOneID, Queryid: jsonID, Addedversion: 1, Name: "example_key", diff --git a/internal/database/repository/result.sql.go b/internal/database/repository/result.sql.go index a3143323..8d32766b 100644 --- a/internal/database/repository/result.sql.go +++ b/internal/database/repository/result.sql.go @@ -11,34 +11,91 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const addResult = `-- name: AddResult :one +INSERT INTO results (queryId, value, textEntryId, queryVersion) VALUES ($1, $2, $3, $4) returning id +` + +type AddResultParams struct { + Queryid pgtype.UUID `db:"queryid"` + Value string `db:"value"` + Textentryid pgtype.UUID `db:"textentryid"` + Queryversion int32 `db:"queryversion"` +} + +// AddResult +// +// INSERT INTO results (queryId, value, textEntryId, queryVersion) VALUES ($1, $2, $3, $4) returning id +func (q *Queries) AddResult(ctx context.Context, arg *AddResultParams) (pgtype.UUID, error) { + row := q.db.QueryRow(ctx, addResult, + arg.Queryid, + arg.Value, + arg.Textentryid, + arg.Queryversion, + ) + var id pgtype.UUID + err := row.Scan(&id) + return id, err +} + +const addResultDependency = `-- name: AddResultDependency :exec +INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2) +` + +type AddResultDependencyParams struct { + Resultid pgtype.UUID `db:"resultid"` + Requiredresultid pgtype.UUID `db:"requiredresultid"` +} + +// AddResultDependency +// +// INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2) +func (q *Queries) AddResultDependency(ctx context.Context, arg *AddResultDependencyParams) error { + _, err := q.db.Exec(ctx, addResultDependency, arg.Resultid, arg.Requiredresultid) + return err +} + const getResultValueWithVersion = `-- name: GetResultValueWithVersion :one -SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5 +WITH doc as ( + SELECT id, jobId + FROM documents + WHERE id = $3 +) +SELECT r.id, r.value + FROM doc as d + JOIN currentTextEntries as cte on cte.documentId = d.id + LEFT JOIN results as r + on r.queryId = $1 + and r.queryVersion = $2 + and r.textEntryId = cte.id ` type GetResultValueWithVersionParams struct { Queryid pgtype.UUID `db:"queryid"` - Queryversion int32 `db:"queryversion"` + Queryversion *int32 `db:"queryversion"` Documentid pgtype.UUID `db:"documentid"` - Cleanversion int32 `db:"cleanversion"` - Textversion int32 `db:"textversion"` } type GetResultValueWithVersionRow struct { ID pgtype.UUID `db:"id"` - Value string `db:"value"` + Value *string `db:"value"` } // GetResultValueWithVersion // -// SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5 +// WITH doc as ( +// SELECT id, jobId +// FROM documents +// WHERE id = $3 +// ) +// SELECT r.id, r.value +// FROM doc as d +// JOIN currentTextEntries as cte on cte.documentId = d.id +// LEFT JOIN results as r +// on r.queryId = $1 +// and r.queryVersion = $2 +// and r.textEntryId = cte.id func (q *Queries) GetResultValueWithVersion(ctx context.Context, arg *GetResultValueWithVersionParams) (*GetResultValueWithVersionRow, error) { - row := q.db.QueryRow(ctx, getResultValueWithVersion, - arg.Queryid, - arg.Queryversion, - arg.Documentid, - arg.Cleanversion, - arg.Textversion, - ) + row := q.db.QueryRow(ctx, getResultValueWithVersion, arg.Queryid, arg.Queryversion, arg.Documentid) var i GetResultValueWithVersionRow err := row.Scan(&i.ID, &i.Value) return &i, err @@ -46,131 +103,110 @@ func (q *Queries) GetResultValueWithVersion(ctx context.Context, arg *GetResultV const listQueryRequirementValues = `-- name: ListQueryRequirementValues :many WITH reqQueries as ( - SELECT q.id as queryId, av.id as activeVersion, q.type + SELECT av.queryId, av.activeVersion, q.type FROM requiredQueries as rq - JOIN queries as q on q.id = rq.requiredQueryId - LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId - WHERE rq.queryId = $2 - and isInVersion($3, rq.addedVersion, rq.removedVersion) + JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId + JOIN queries as q on q.id = av.queryId + WHERE rq.queryId = $1 + and isInVersion($2, rq.addedVersion, rq.removedVersion) ), docs as ( SELECT id, jobId FROM documents - WHERE id = $1 + WHERE id = $3 ), codeVersions as ( - SELECT + SELECT d.id as documentId, - coalesce(ccv.versionId, 1) as minCleanVersion, - coalesce(ctv.versionId, 1) as minTextVersion + mcv.minCleanVersion, + mtv.minTextVersion FROM docs as d - LEFT JOIN collectors as c on c.jobId = d.jobId - LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId - LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId - and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) - LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId - and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion) - LIMIT 1 + JOIN currentCollectorMinTextVersions as mtv on mtv.jobId = d.jobId + JOIN currentCollectorMinCleanVersions as mcv on mcv.jobId = d.jobId ), latestVersions AS ( - SELECT + SELECT + r.id, rq.queryId, rq.type, r.queryVersion, - r.cleanVersion, - r.textVersion, + r.textEntryId, + r.value, ROW_NUMBER() OVER ( - PARTITION BY rq.queryId - ORDER BY r.cleanVersion DESC, r.textVersion DESC + PARTITION BY r.queryId + ORDER BY r.id DESC ) as rowNumber FROM reqQueries as rq + JOIN currentTextEntries as cte on cte.documentId = $3 LEFT JOIN results as r ON rq.queryId = r.queryId - and r.documentId = $1 and r.queryVersion = rq.activeVersion - JOIN codeVersions as ccv on ccv.documentId = r.documentId - and r.cleanVersion >= ccv.minCleanVersion - and r.textVersion >= ccv.minTextVersion + and cte.id = r.textEntryId ) -SELECT DISTINCT lv.queryId, lv.type, r.value -FROM latestVersions as lv -JOIN results as r ON r.queryId = lv.queryId - and r.documentId = $1 - and r.queryVersion = lv.queryVersion - and r.cleanVersion = lv.cleanVersion - and r.textVersion = lv.textVersion - and lv.rowNumber = 1 +SELECT DISTINCT id, queryId, type, value +FROM latestVersions +WHERE rowNumber = 1 ` type ListQueryRequirementValuesParams struct { - Documentid pgtype.UUID `db:"documentid"` Queryid pgtype.UUID `db:"queryid"` - Version int32 `db:"version"` + Version *int32 `db:"version"` + Documentid pgtype.UUID `db:"documentid"` } type ListQueryRequirementValuesRow struct { + ID pgtype.UUID `db:"id"` Queryid pgtype.UUID `db:"queryid"` Type Querytype `db:"type"` - Value string `db:"value"` + Value *string `db:"value"` } // ListQueryRequirementValues // // WITH reqQueries as ( -// SELECT q.id as queryId, av.id as activeVersion, q.type +// SELECT av.queryId, av.activeVersion, q.type // FROM requiredQueries as rq -// JOIN queries as q on q.id = rq.requiredQueryId -// LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId -// WHERE rq.queryId = $2 -// and isInVersion($3, rq.addedVersion, rq.removedVersion) +// JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId +// JOIN queries as q on q.id = av.queryId +// WHERE rq.queryId = $1 +// and isInVersion($2, rq.addedVersion, rq.removedVersion) // ), // docs as ( // SELECT id, jobId // FROM documents -// WHERE id = $1 +// WHERE id = $3 // ), // codeVersions as ( // SELECT // d.id as documentId, -// coalesce(ccv.versionId, 1) as minCleanVersion, -// coalesce(ctv.versionId, 1) as minTextVersion +// mcv.minCleanVersion, +// mtv.minTextVersion // FROM docs as d -// LEFT JOIN collectors as c on c.jobId = d.jobId -// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId -// LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId -// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) -// LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId -// and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion) -// LIMIT 1 +// JOIN currentCollectorMinTextVersions as mtv on mtv.jobId = d.jobId +// JOIN currentCollectorMinCleanVersions as mcv on mcv.jobId = d.jobId // ), // latestVersions AS ( // SELECT +// r.id, // rq.queryId, // rq.type, // r.queryVersion, -// r.cleanVersion, -// r.textVersion, +// r.textEntryId, +// r.value, // ROW_NUMBER() OVER ( -// PARTITION BY rq.queryId -// ORDER BY r.cleanVersion DESC, r.textVersion DESC +// PARTITION BY r.queryId +// ORDER BY r.id DESC // ) as rowNumber // FROM reqQueries as rq +// JOIN currentTextEntries as cte on cte.documentId = $3 // LEFT JOIN results as r ON rq.queryId = r.queryId -// and r.documentId = $1 // and r.queryVersion = rq.activeVersion -// JOIN codeVersions as ccv on ccv.documentId = r.documentId -// and r.cleanVersion >= ccv.minCleanVersion -// and r.textVersion >= ccv.minTextVersion +// and cte.id = r.textEntryId // ) -// SELECT DISTINCT lv.queryId, lv.type, r.value -// FROM latestVersions as lv -// JOIN results as r ON r.queryId = lv.queryId -// and r.documentId = $1 -// and r.queryVersion = lv.queryVersion -// and r.cleanVersion = lv.cleanVersion -// and r.textVersion = lv.textVersion -// and lv.rowNumber = 1 +// SELECT DISTINCT id, queryId, type, value +// FROM latestVersions +// WHERE rowNumber = 1 func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQueryRequirementValuesParams) ([]*ListQueryRequirementValuesRow, error) { - rows, err := q.db.Query(ctx, listQueryRequirementValues, arg.Documentid, arg.Queryid, arg.Version) + rows, err := q.db.Query(ctx, listQueryRequirementValues, arg.Queryid, arg.Version, arg.Documentid) if err != nil { return nil, err } @@ -178,7 +214,12 @@ func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQuery items := []*ListQueryRequirementValuesRow{} for rows.Next() { var i ListQueryRequirementValuesRow - if err := rows.Scan(&i.Queryid, &i.Type, &i.Value); err != nil { + if err := rows.Scan( + &i.ID, + &i.Queryid, + &i.Type, + &i.Value, + ); err != nil { return nil, err } items = append(items, &i) @@ -194,14 +235,14 @@ WITH docs as ( SELECT id, jobId from documents where id = $1 ), unsyncedQueries AS ( - SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id + SELECT DISTINCT dt.queryId, dt.requiredIds from docs as d JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId - JOIN fullActiveCollectors as c on c.id = dt.collectorId - LEFT JOIN results as r on r.queryId = dt.queryId - and r.documentId = d.id + JOIN currentTextEntries as cte on cte.documentId = d.id + LEFT JOIN results as r + on r.queryId = dt.queryId and r.queryVersion = dt.queryVersion - and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion + and cte.id = r.textEntryId where r.value is null ) SELECT DISTINCT queryId FROM unsyncedQueries as baseuq @@ -216,14 +257,14 @@ SELECT DISTINCT queryId FROM unsyncedQueries as baseuq // SELECT id, jobId from documents where id = $1 // ), // unsyncedQueries AS ( -// SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id +// SELECT DISTINCT dt.queryId, dt.requiredIds // from docs as d // JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId -// JOIN fullActiveCollectors as c on c.id = dt.collectorId -// LEFT JOIN results as r on r.queryId = dt.queryId -// and r.documentId = d.id +// JOIN currentTextEntries as cte on cte.documentId = d.id +// LEFT JOIN results as r +// on r.queryId = dt.queryId // and r.queryVersion = dt.queryVersion -// and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion +// and cte.id = r.textEntryId // where r.value is null // ) // SELECT DISTINCT queryId FROM unsyncedQueries as baseuq @@ -249,31 +290,3 @@ func (q *Queries) ListUnsyncedNoDepsQueriesByDocId(ctx context.Context, dollar_1 } return items, nil } - -const setResult = `-- name: SetResult :exec -INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) -` - -type SetResultParams struct { - Queryid pgtype.UUID `db:"queryid"` - Documentid pgtype.UUID `db:"documentid"` - Value string `db:"value"` - Cleanversion int32 `db:"cleanversion"` - Textversion int32 `db:"textversion"` - Queryversion int32 `db:"queryversion"` -} - -// SetResult -// -// INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) -func (q *Queries) SetResult(ctx context.Context, arg *SetResultParams) error { - _, err := q.db.Exec(ctx, setResult, - arg.Queryid, - arg.Documentid, - arg.Value, - arg.Cleanversion, - arg.Textversion, - arg.Queryversion, - ) - return err -} diff --git a/internal/database/repository/result_test.go b/internal/database/repository/result_test.go index c019b523..0eb4c508 100644 --- a/internal/database/repository/result_test.go +++ b/internal/database/repository/result_test.go @@ -44,38 +44,98 @@ func TestResults(t *testing.T) { assert.NoError(t, err) jobId, err := queries.CreateJob(ctx, clientId) assert.NoError(t, err) + + issynced, err := queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, issynced) + documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ Jobid: jobId, Hash: "example_hash", }) assert.NoError(t, err) - jsonQuery, err := queries.GetQuery(ctx, jsonQueryID) + issynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, issynced) + + version, err := queries.AddLatestCollectorVersion(ctx, jobId) + assert.NoError(t, err) + err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ + Versionid: version, + Jobid: jobId, + }) assert.NoError(t, err) - cleanVersion := int32(1) - textVersion := int32(1) + issynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, issynced) + + err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ + Documentid: documentID, + Version: 1, + Bucket: "hi", + Key: "hello", + }) + assert.NoError(t, err) + + issynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, issynced) + + cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID) + assert.NoError(t, err) + + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Bucket: "hi", + Key: "hello", + Cleanentryid: cleanentry.ID, + }) + assert.NoError(t, err) + + issynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, issynced) + + textentry, err := queries.GetDocumentTextEntry(ctx, documentID) + assert.NoError(t, err) + + err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + Jobid: jobId, + Queryid: jsonQueryID, + Addedversion: 1, + Name: "example_key", + }) + assert.NoError(t, err) + + issynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.False(t, issynced) + jsonResultValue := "example_value" - err = queries.SetResult(ctx, &repository.SetResultParams{ + _, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, - Documentid: documentID, Value: jsonResultValue, - Cleanversion: cleanVersion, - Textversion: textVersion, - Queryversion: jsonQuery.Activeversion, + Textentryid: textentry.ID, + Queryversion: 1, }) assert.NoError(t, err) + issynced, err = queries.IsJobSynced(ctx, jobId) + assert.NoError(t, err) + assert.True(t, issynced) + + qv := int32(1) res, err := queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{ Queryid: jsonQueryID, - Queryversion: jsonQuery.Activeversion, + Queryversion: &qv, Documentid: documentID, - Cleanversion: cleanVersion, - Textversion: textVersion, }) assert.NoError(t, err) - assert.EqualExportedValues(t, jsonResultValue, res.Value) + assert.NotNil(t, res.Value) + assert.Equal(t, jsonResultValue, *res.Value) } func TestResultValues(t *testing.T) { @@ -147,58 +207,79 @@ func TestResultValues(t *testing.T) { qResults, err := queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ Queryid: jsonQueryID, Documentid: documentID, - Version: jsonVersion, + Version: &jsonVersion, }) assert.NoError(t, err) assert.Len(t, qResults, 0) qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ Queryid: jsonQueryID, Documentid: documentTwoID, - Version: jsonVersion, + Version: &jsonVersion, }) assert.NoError(t, err) assert.Len(t, qResults, 0) - result := repository.SetResultParams{ + version, err := queries.AddLatestCollectorVersion(ctx, jobId) + assert.NoError(t, err) + err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ + Versionid: version, + Jobid: jobId, + }) + assert.NoError(t, err) + + err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ + Documentid: documentID, + Version: 1, + Bucket: "hi", + Key: "hello", + }) + assert.NoError(t, err) + cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID) + assert.NoError(t, err) + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Bucket: "hi", + Key: "hello", + Cleanentryid: cleanentry.ID, + }) + assert.NoError(t, err) + textentry, err := queries.GetDocumentTextEntry(ctx, documentID) + assert.NoError(t, err) + + result := repository.AddResultParams{ Queryid: contextQueryID, - Documentid: documentID, Value: "context_value_1", - Cleanversion: 1, - Textversion: 2, + Textentryid: textentry.ID, Queryversion: contextQuery.Activeversion, } - err = queries.SetResult(ctx, &result) + _, err = queries.AddResult(ctx, &result) assert.NoError(t, err) qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ Queryid: jsonQueryID, Documentid: documentID, - Version: jsonVersion, + Version: &jsonVersion, }) assert.NoError(t, err) assert.Len(t, qResults, 1) - assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{ - { - Queryid: contextQueryID, - Type: repository.QuerytypeContextFull, - Value: "context_value_1", - }, - }, qResults) + assert.Equal(t, contextQueryID, qResults[0].Queryid) + assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type) + assert.Equal(t, "context_value_1", *qResults[0].Value) + assert.NotEqual(t, pgtype.UUID{}, qResults[0].ID) + assert.True(t, qResults[0].ID.Valid) qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ Queryid: jsonQueryID, Documentid: documentTwoID, - Version: jsonVersion, + Version: &jsonVersion, }) assert.NoError(t, err) assert.Len(t, qResults, 0) - err = queries.SetResult(ctx, &repository.SetResultParams{ + _, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, - Documentid: documentID, Value: "context_value_2", - Cleanversion: 1, - Textversion: 2, + Textentryid: textentry.ID, Queryversion: contextQuery.Activeversion - 1, }) assert.NoError(t, err) @@ -206,24 +287,20 @@ func TestResultValues(t *testing.T) { qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ Queryid: jsonQueryID, Documentid: documentID, - Version: jsonVersion, + Version: &jsonVersion, }) assert.NoError(t, err) assert.Len(t, qResults, 1) - assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{ - { - Queryid: contextQueryID, - Type: repository.QuerytypeContextFull, - Value: "context_value_1", - }, - }, qResults) + assert.Equal(t, contextQueryID, qResults[0].Queryid) + assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type) + assert.Equal(t, "context_value_1", *qResults[0].Value) + assert.NotEqual(t, pgtype.UUID{}, qResults[0].ID) + assert.True(t, qResults[0].ID.Valid) - err = queries.SetResult(ctx, &repository.SetResultParams{ + _, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, - Documentid: documentID, Value: "context_value_3", - Cleanversion: 2, - Textversion: 2, + Textentryid: textentry.ID, Queryversion: contextQuery.Activeversion, }) assert.NoError(t, err) @@ -231,24 +308,20 @@ func TestResultValues(t *testing.T) { qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ Queryid: jsonQueryID, Documentid: documentID, - Version: jsonVersion, + Version: &jsonVersion, }) assert.NoError(t, err) assert.Len(t, qResults, 1) - assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{ - { - Queryid: contextQueryID, - Type: repository.QuerytypeContextFull, - Value: "context_value_3", - }, - }, qResults) + assert.Equal(t, contextQueryID, qResults[0].Queryid) + assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type) + assert.Equal(t, "context_value_3", *qResults[0].Value) + assert.NotEqual(t, pgtype.UUID{}, qResults[0].ID) + assert.True(t, qResults[0].ID.Valid) - err = queries.SetResult(ctx, &repository.SetResultParams{ + _, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, - Documentid: documentID, Value: "json_value_1", - Cleanversion: 1, - Textversion: 2, + Textentryid: textentry.ID, Queryversion: jsonVersion, }) assert.NoError(t, err) @@ -256,17 +329,15 @@ func TestResultValues(t *testing.T) { qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ Queryid: jsonQueryID, Documentid: documentID, - Version: jsonVersion, + Version: &jsonVersion, }) assert.NoError(t, err) assert.Len(t, qResults, 1) - assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{ - { - Queryid: contextQueryID, - Type: repository.QuerytypeContextFull, - Value: "context_value_3", - }, - }, qResults) + assert.Equal(t, contextQueryID, qResults[0].Queryid) + assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type) + assert.Equal(t, "context_value_3", *qResults[0].Value) + assert.NotEqual(t, pgtype.UUID{}, qResults[0].ID) + assert.True(t, qResults[0].ID.Valid) } func TestUnsyncedNoDepsQueries(t *testing.T) { @@ -290,13 +361,11 @@ func TestUnsyncedNoDepsQueries(t *testing.T) { assert.NoError(t, err) jobId, err := queries.CreateJob(ctx, clientId) assert.NoError(t, err) - collectorId, err := queries.CreateCollector(ctx, jobId) - assert.NoError(t, err) - _, err = queries.AddLatestCollectorVersion(ctx, collectorId) + version, err := queries.AddLatestCollectorVersion(ctx, jobId) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Collectorid: collectorId, - Versionid: 1, + Versionid: version, + Jobid: jobId, }) assert.NoError(t, err) documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ @@ -336,7 +405,7 @@ func TestUnsyncedNoDepsQueries(t *testing.T) { assert.NoError(t, err) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Collectorid: collectorId, + Jobid: jobId, Name: "example_name", Queryid: jsonQueryID, Addedversion: 1, @@ -345,19 +414,52 @@ func TestUnsyncedNoDepsQueries(t *testing.T) { qs, err := queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID) assert.NoError(t, err) + assert.Len(t, qs, 0) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID) + assert.NoError(t, err) + assert.Len(t, qs, 0) + + err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ + Documentid: documentID, + Version: 1, + Bucket: "hi", + Key: "hello", + }) + assert.NoError(t, err) + + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID) + assert.NoError(t, err) + assert.Len(t, qs, 0) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID) + assert.NoError(t, err) + assert.Len(t, qs, 0) + + cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID) + assert.NoError(t, err) + + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Bucket: "hi", + Key: "hello", + Cleanentryid: cleanentry.ID, + }) + assert.NoError(t, err) + + textentry, err := queries.GetDocumentTextEntry(ctx, documentID) + assert.NoError(t, err) + + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID) + assert.NoError(t, err) assert.Len(t, qs, 1) assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs) qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID) assert.NoError(t, err) - assert.Len(t, qs, 1) - assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs) + assert.Len(t, qs, 0) - err = queries.SetResult(ctx, &repository.SetResultParams{ + _, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, - Documentid: documentID, Value: "context_value", - Cleanversion: 1, - Textversion: 2, + Textentryid: textentry.ID, Queryversion: 1, }) assert.NoError(t, err) @@ -368,15 +470,12 @@ func TestUnsyncedNoDepsQueries(t *testing.T) { assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, qs) qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID) assert.NoError(t, err) - assert.Len(t, qs, 1) - assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs) + assert.Len(t, qs, 0) - err = queries.SetResult(ctx, &repository.SetResultParams{ + _, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, - Documentid: documentID, Value: "context_value", - Cleanversion: 1, - Textversion: 2, + Textentryid: textentry.ID, Queryversion: 1, }) assert.NoError(t, err) @@ -386,15 +485,41 @@ func TestUnsyncedNoDepsQueries(t *testing.T) { assert.Len(t, qs, 0) qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID) assert.NoError(t, err) - assert.Len(t, qs, 1) - assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs) + assert.Len(t, qs, 0) - err = queries.SetResult(ctx, &repository.SetResultParams{ + err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ + Documentid: documentTwoID, + Version: 1, + Bucket: "hi", + Key: "hello", + }) + assert.NoError(t, err) + + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID) + assert.NoError(t, err) + assert.Len(t, qs, 0) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID) + assert.NoError(t, err) + assert.Len(t, qs, 0) + + cleantwoentry, err := queries.GetDocumentCleanEntry(ctx, documentTwoID) + assert.NoError(t, err) + + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Bucket: "hi", + Key: "hello", + Cleanentryid: cleantwoentry.ID, + }) + assert.NoError(t, err) + + texttwoentry, err := queries.GetDocumentTextEntry(ctx, documentTwoID) + assert.NoError(t, err) + + _, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, - Documentid: documentTwoID, Value: "context_value", - Cleanversion: 1, - Textversion: 2, + Textentryid: texttwoentry.ID, Queryversion: 1, }) assert.NoError(t, err) @@ -407,12 +532,10 @@ func TestUnsyncedNoDepsQueries(t *testing.T) { assert.Len(t, qs, 1) assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, qs) - err = queries.SetResult(ctx, &repository.SetResultParams{ + _, err = queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, - Documentid: documentTwoID, Value: "context_value", - Cleanversion: 1, - Textversion: 2, + Textentryid: texttwoentry.ID, Queryversion: 1, }) assert.NoError(t, err) diff --git a/internal/database/repository/text.sql.go b/internal/database/repository/text.sql.go index 502eb486..e057e6df 100644 --- a/internal/database/repository/text.sql.go +++ b/internal/database/repository/text.sql.go @@ -12,95 +12,64 @@ import ( ) const addDocumentTextEntry = `-- name: AddDocumentTextEntry :exec -INSERT INTO documentTextExtractions (documentId, version, bucket, key) VALUES ($1, $2, $3, $4) +INSERT INTO documentTextExtractions (version, bucket, key, cleanEntryId) VALUES ($1, $2, $3, $4) ` type AddDocumentTextEntryParams struct { - Documentid pgtype.UUID `db:"documentid"` - Version int32 `db:"version"` - Bucket string `db:"bucket"` - Key string `db:"key"` + Version int32 `db:"version"` + Bucket string `db:"bucket"` + Key string `db:"key"` + Cleanentryid pgtype.UUID `db:"cleanentryid"` } // AddDocumentTextEntry // -// INSERT INTO documentTextExtractions (documentId, version, bucket, key) VALUES ($1, $2, $3, $4) +// INSERT INTO documentTextExtractions (version, bucket, key, cleanEntryId) VALUES ($1, $2, $3, $4) func (q *Queries) AddDocumentTextEntry(ctx context.Context, arg *AddDocumentTextEntryParams) error { _, err := q.db.Exec(ctx, addDocumentTextEntry, - arg.Documentid, arg.Version, arg.Bucket, arg.Key, + arg.Cleanentryid, ) return err } const getDocumentTextEntry = `-- name: GetDocumentTextEntry :one -SELECT dc.documentId, dc.bucket, dc.key, dc.version - FROM documentTextExtractions AS dc - JOIN documents AS d ON d.id = dc.documentId - LEFT JOIN collectors as c ON d.jobId = c.jobId - LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId - LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId - and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) - WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1) - ORDER BY d.id DESC LIMIT 1 +SELECT id, documentId, bucket, key, version, cleanEntryId + FROM currentTextEntries + WHERE documentId = $1 ` -type GetDocumentTextEntryRow struct { - Documentid pgtype.UUID `db:"documentid"` - Bucket string `db:"bucket"` - Key string `db:"key"` - Version int32 `db:"version"` -} - // GetDocumentTextEntry // -// SELECT dc.documentId, dc.bucket, dc.key, dc.version -// FROM documentTextExtractions AS dc -// JOIN documents AS d ON d.id = dc.documentId -// LEFT JOIN collectors as c ON d.jobId = c.jobId -// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId -// LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId -// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) -// WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1) -// ORDER BY d.id DESC LIMIT 1 -func (q *Queries) GetDocumentTextEntry(ctx context.Context, documentid pgtype.UUID) (*GetDocumentTextEntryRow, error) { +// SELECT id, documentId, bucket, key, version, cleanEntryId +// FROM currentTextEntries +// WHERE documentId = $1 +func (q *Queries) GetDocumentTextEntry(ctx context.Context, documentid pgtype.UUID) (*Currenttextentry, error) { row := q.db.QueryRow(ctx, getDocumentTextEntry, documentid) - var i GetDocumentTextEntryRow + var i Currenttextentry err := row.Scan( + &i.ID, &i.Documentid, &i.Bucket, &i.Key, &i.Version, + &i.Cleanentryid, ) return &i, err } const isDocumentTextExtracted = `-- name: IsDocumentTextExtracted :one SELECT EXISTS( - SELECT 1 - FROM documentTextExtractions AS dc - JOIN documents AS d ON d.id = dc.documentId - LEFT JOIN collectors as c ON d.jobId = c.jobId - LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId - LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId - and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) - WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1) + SELECT 1 FROM currentTextEntries WHERE documentId = $1 ) ` // IsDocumentTextExtracted // // SELECT EXISTS( -// SELECT 1 -// FROM documentTextExtractions AS dc -// JOIN documents AS d ON d.id = dc.documentId -// LEFT JOIN collectors as c ON d.jobId = c.jobId -// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId -// LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId -// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion) -// WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1) +// SELECT 1 FROM currentTextEntries WHERE documentId = $1 // ) func (q *Queries) IsDocumentTextExtracted(ctx context.Context, documentid pgtype.UUID) (bool, error) { row := q.db.QueryRow(ctx, isDocumentTextExtracted, documentid) diff --git a/internal/database/repository/text_test.go b/internal/database/repository/text_test.go index 53c0d086..7660685f 100644 --- a/internal/database/repository/text_test.go +++ b/internal/database/repository/text_test.go @@ -9,6 +9,7 @@ import ( "queryorchestration/internal/test" "testing" + "github.com/jackc/pgx/v5/pgtype" "github.com/stretchr/testify/assert" ) @@ -21,11 +22,11 @@ func TestTextExtraction(t *testing.T) { cfg := &serviceconfig.BaseConfig{} test.SetCfgProvider(t, cfg) cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../..")) - _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + _, textup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ Cfg: cfg, RunMigrations: true, }) - defer cleanup() + defer textup() queries := cfg.GetDBQueries() @@ -42,16 +43,9 @@ func TestTextExtraction(t *testing.T) { assert.NoError(t, err) assert.NotEmpty(t, id) - isextract, err := queries.IsDocumentTextExtracted(ctx, id) - assert.NoError(t, err) - assert.False(t, isextract) - - _, err = queries.GetDocumentTextEntry(ctx, id) - assert.Error(t, err) - bucket := "example_bucket" key := "example_key" - err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ Documentid: id, Version: 1, Bucket: bucket, @@ -59,16 +53,31 @@ func TestTextExtraction(t *testing.T) { }) assert.NoError(t, err) + cleanentry, err := queries.GetDocumentCleanEntry(ctx, id) + assert.NoError(t, err) + + isextract, err := queries.IsDocumentTextExtracted(ctx, id) + assert.NoError(t, err) + assert.False(t, isextract) + + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Bucket: bucket, + Key: key, + Cleanentryid: cleanentry.ID, + }) + assert.NoError(t, err) + isextract, err = queries.IsDocumentTextExtracted(ctx, id) assert.NoError(t, err) assert.True(t, isextract) - clean, err := queries.GetDocumentTextEntry(ctx, id) + text, err := queries.GetDocumentTextEntry(ctx, id) assert.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentTextEntryRow{ - Documentid: id, - Bucket: bucket, - Key: key, - Version: 1, - }, clean) + assert.Equal(t, id, text.Documentid) + assert.Equal(t, bucket, text.Bucket) + assert.Equal(t, key, text.Key) + assert.Equal(t, int32(1), text.Version) + assert.NotEqual(t, pgtype.UUID{}, text.ID) + assert.True(t, text.ID.Valid) } diff --git a/internal/document/text/create_test.go b/internal/document/text/create_test.go index 8c3e3669..dd0e8265 100644 --- a/internal/document/text/create_test.go +++ b/internal/document/text/create_test.go @@ -56,11 +56,12 @@ func TestCreate(t *testing.T) { pgxmock.NewRows([]string{"isextracted"}). AddRow(false), ) + cleanId := database.MustToDBUUID(uuid.New()) pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows( - pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). - AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key, int32(1)), + pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version"}). + AddRow(cleanId, database.MustToDBUUID(id), inloc.Bucket, inloc.Key, int32(1)), ) - pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(database.MustToDBUUID(id), int32(1), inloc.Bucket, inloc.Key). + pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(int32(1), inloc.Bucket, inloc.Key, cleanId). WillReturnResult(pgxmock.NewResult("", 1)) mockSQS.EXPECT(). diff --git a/internal/document/text/extract.go b/internal/document/text/extract.go index e43be303..94bb6259 100644 --- a/internal/document/text/extract.go +++ b/internal/document/text/extract.go @@ -41,10 +41,10 @@ func (s *Service) extract(ctx context.Context, id uuid.UUID) error { version := s.svc.Version.GetVersion() err = s.cfg.GetDBQueries().AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ - Documentid: docId, - Version: version, - Bucket: outLocation.Bucket, - Key: outLocation.Key, + Version: version, + Bucket: outLocation.Bucket, + Key: outLocation.Key, + Cleanentryid: entry.ID, }) if err != nil { return err diff --git a/internal/document/text/extract_test.go b/internal/document/text/extract_test.go index b25acb94..73c10ca3 100644 --- a/internal/document/text/extract_test.go +++ b/internal/document/text/extract_test.go @@ -37,11 +37,12 @@ func TestExtract(t *testing.T) { Key: "/i/am/here", } + cleanId := database.MustToDBUUID(uuid.New()) pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows( - pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). - AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key, int32(1)), + pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version"}). + AddRow(cleanId, database.MustToDBUUID(id), inloc.Bucket, inloc.Key, int32(1)), ) - pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(database.MustToDBUUID(id), int32(1), inloc.Bucket, inloc.Key). + pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(int32(1), inloc.Bucket, inloc.Key, cleanId). WillReturnResult(pgxmock.NewResult("", 1)) err = svc.extract(ctx, id) diff --git a/internal/job/collector/create.go b/internal/job/collector/create.go index c46d80a9..ea8f526c 100644 --- a/internal/job/collector/create.go +++ b/internal/job/collector/create.go @@ -18,18 +18,13 @@ type CreateParams struct { Fields *map[string]uuid.UUID } -func (s *Service) Create(ctx context.Context, params *CreateParams) (uuid.UUID, error) { +func (s *Service) Create(ctx context.Context, params *CreateParams) error { dbparams, err := s.getCreateParams(ctx, params) if err != nil { - return uuid.Nil, err + return err } - id, err := s.submitCreate(ctx, dbparams) - if err != nil { - return uuid.Nil, err - } - - return id, nil + return s.submitCreate(ctx, dbparams) } type dbCreateParams struct { @@ -102,23 +97,16 @@ func (s *Service) NormalizeFieldsToDB(ctx context.Context, ofields *map[string]u return &dbm, nil } -func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) (uuid.UUID, error) { - var dbID pgtype.UUID - err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error { - dID, err := qtx.CreateCollector(ctx, params.JobID) - if err != nil { - return err - } - dbID = dID - - version, err := qtx.AddLatestCollectorVersion(ctx, dbID) +func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) error { + return s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error { + version, err := qtx.AddLatestCollectorVersion(ctx, params.JobID) if err != nil { return err } err = qtx.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Collectorid: dbID, - Versionid: version, + Jobid: params.JobID, + Versionid: version, }) if err != nil { return err @@ -126,7 +114,7 @@ func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) (uui if params.MinCleanVersion != nil { err = qtx.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{ - Collectorid: dbID, + Jobid: params.JobID, Versionid: *params.MinCleanVersion, Addedversion: version, }) @@ -137,7 +125,7 @@ func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) (uui if params.MinTextVersion != nil { err = qtx.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{ - Collectorid: dbID, + Jobid: params.JobID, Versionid: *params.MinTextVersion, Addedversion: version, }) @@ -149,7 +137,7 @@ func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) (uui if params.Fields != nil { for key, field := range *params.Fields { err = qtx.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Collectorid: dbID, + Jobid: params.JobID, Name: key, Queryid: field, Addedversion: version, @@ -162,14 +150,4 @@ func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) (uui return nil }) - if err != nil { - return uuid.Nil, err - } - - id, err := database.ToUUID(dbID) - if err != nil { - return uuid.Nil, err - } - - return id, nil } diff --git a/internal/job/collector/create_test.go b/internal/job/collector/create_test.go index d8e93e4e..2c4fc125 100644 --- a/internal/job/collector/create_test.go +++ b/internal/job/collector/create_test.go @@ -27,7 +27,6 @@ func TestCreate(t *testing.T) { svc := collector.New(cfg, &collector.Services{}) - id := uuid.New() minCleanV := int32(1) minTextV := int32(1) create := collector.CreateParams{ @@ -44,25 +43,20 @@ func TestCreate(t *testing.T) { AddRow(true), ) pool.ExpectBeginTx(pgx.TxOptions{}) - pool.ExpectQuery("name: CreateCollector :one").WithArgs(database.MustToDBUUID(create.JobID)).WillReturnRows( - pgxmock.NewRows([]string{"id"}). - AddRow(database.MustToDBUUID(id)), - ) - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows( + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(create.JobID)).WillReturnRows( pgxmock.NewRows([]string{"version"}). AddRow(int32(1)), ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(create.JobID), int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1), *create.MinCleanVersion). + pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(create.JobID), int32(1), *create.MinCleanVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1), *create.MinTextVersion). + pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(create.JobID), int32(1), *create.MinTextVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(id), "example_key", database.MustToDBUUID((*create.Fields)["example_key"]), int32(1)). + pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(create.JobID), "example_key", database.MustToDBUUID((*create.Fields)["example_key"]), int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() - aid, err := svc.Create(ctx, &create) + err = svc.Create(ctx, &create) assert.NoError(t, err) - assert.Equal(t, id, aid) } diff --git a/internal/job/collector/createprivate_test.go b/internal/job/collector/createprivate_test.go index 6d5562b7..906997c6 100644 --- a/internal/job/collector/createprivate_test.go +++ b/internal/job/collector/createprivate_test.go @@ -81,7 +81,6 @@ func TestSubmitCreate(t *testing.T) { minCleanV := int32(2) minTextV := int32(4) - id := uuid.New() params := dbCreateParams{ JobID: database.MustToDBUUID(uuid.New()), MinCleanVersion: &minCleanV, @@ -92,27 +91,22 @@ func TestSubmitCreate(t *testing.T) { } pool.ExpectBeginTx(pgx.TxOptions{}) - pool.ExpectQuery("name: CreateCollector :one").WithArgs(params.JobID).WillReturnRows( - pgxmock.NewRows([]string{"id"}). - AddRow(database.MustToDBUUID(id)), - ) - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows( + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(params.JobID).WillReturnRows( pgxmock.NewRows([]string{"version"}). AddRow(int32(1)), ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(params.JobID, int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1), *params.MinCleanVersion). + pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(params.JobID, int32(1), *params.MinCleanVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1), *params.MinTextVersion). + pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(params.JobID, int32(1), *params.MinTextVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(id), "example_key", (*params.Fields)["example_key"], int32(1)). + pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(params.JobID, "example_key", (*params.Fields)["example_key"], int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() - aid, err := svc.submitCreate(ctx, ¶ms) + err = svc.submitCreate(ctx, ¶ms) assert.NoError(t, err) - assert.Equal(t, id, aid) } func TestNormalizeFieldsToDB(t *testing.T) { diff --git a/internal/job/collector/get_test.go b/internal/job/collector/get_test.go index 36152163..968dc1e3 100644 --- a/internal/job/collector/get_test.go +++ b/internal/job/collector/get_test.go @@ -30,7 +30,6 @@ func TestGet(t *testing.T) { minCleanV := int32(2) minTextV := int32(4) ogc := collector.Collector{ - ID: uuid.New(), JobID: uuid.New(), MinCleanVersion: minCleanV, MinTextVersion: minTextV, @@ -39,13 +38,13 @@ func TestGet(t *testing.T) { }, } - pool.ExpectQuery("name: GetCollector :one").WithArgs(database.MustToDBUUID(ogc.ID)). + pool.ExpectQuery("name: GetCollector :one").WithArgs(database.MustToDBUUID(ogc.JobID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(ogc.ID), database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))), + pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). + AddRow(database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))), ) - coll, err := svc.Get(ctx, ogc.ID) + coll, err := svc.Get(ctx, ogc.JobID) assert.NoError(t, err) assert.EqualExportedValues(t, ogc, *coll) } @@ -66,7 +65,6 @@ func TestGetByJobID(t *testing.T) { minCleanV := int32(2) minTextV := int32(4) ogc := collector.Collector{ - ID: uuid.New(), JobID: uuid.New(), MinCleanVersion: minCleanV, MinTextVersion: minTextV, @@ -77,8 +75,8 @@ func TestGetByJobID(t *testing.T) { pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(ogc.JobID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(ogc.ID), database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))), + pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). + AddRow(database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))), ) coll, err := svc.GetByJobID(ctx, ogc.JobID) diff --git a/internal/job/collector/parse.go b/internal/job/collector/parse.go index dc5445dd..3e2a00ac 100644 --- a/internal/job/collector/parse.go +++ b/internal/job/collector/parse.go @@ -26,7 +26,6 @@ func parseDBCollector(c *repository.Fullactivecollector) (*Collector, error) { } return &Collector{ - ID: database.MustToUUID(c.ID), JobID: database.MustToUUID(c.Jobid), MinCleanVersion: c.Mincleanversion, MinTextVersion: c.Mintextversion, diff --git a/internal/job/collector/parse_test.go b/internal/job/collector/parse_test.go index bb8632c8..919a3520 100644 --- a/internal/job/collector/parse_test.go +++ b/internal/job/collector/parse_test.go @@ -18,7 +18,6 @@ func TestParseDBCollector(t *testing.T) { minCleanV := int32(1) minTextV := int32(2) ogc := Collector{ - ID: uuid.New(), JobID: uuid.New(), MinCleanVersion: minCleanV, MinTextVersion: minTextV, @@ -27,7 +26,6 @@ func TestParseDBCollector(t *testing.T) { }, } c, err = parseDBCollector(&repository.Fullactivecollector{ - ID: database.MustToDBUUID(ogc.ID), Jobid: database.MustToDBUUID(ogc.JobID), Mincleanversion: ogc.MinCleanVersion, Mintextversion: ogc.MinTextVersion, @@ -39,7 +37,6 @@ func TestParseDBCollector(t *testing.T) { ogc.MinCleanVersion = 0 ogc.MinTextVersion = 0 c, err = parseDBCollector(&repository.Fullactivecollector{ - ID: database.MustToDBUUID(ogc.ID), Jobid: database.MustToDBUUID(ogc.JobID), Fields: []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"])), }) diff --git a/internal/job/collector/service.go b/internal/job/collector/service.go index e9ddded4..0f665833 100644 --- a/internal/job/collector/service.go +++ b/internal/job/collector/service.go @@ -9,7 +9,6 @@ import ( ) type Collector struct { - ID uuid.UUID JobID uuid.UUID MinCleanVersion int32 MinTextVersion int32 diff --git a/internal/job/collector/update/update.go b/internal/job/collector/update/update.go index 9e25730f..8064bd4d 100644 --- a/internal/job/collector/update/update.go +++ b/internal/job/collector/update/update.go @@ -183,8 +183,6 @@ func (s *Service) normalizeUpdateFieldsToDB(ctx context.Context, current map[str func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector, params *dbUpdateParams) error { err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error { - id := database.MustToDBUUID(current.ID) - activeName, err := validation.GetFieldName(params, params.ActiveVersion) if err != nil { return err @@ -193,7 +191,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector var latestVersion int32 if !onlyactive { - latestVersion, err = qtx.AddLatestCollectorVersion(ctx, id) + latestVersion, err = qtx.AddLatestCollectorVersion(ctx, params.JobID) if err != nil { return err } @@ -201,8 +199,8 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector if params.ActiveVersion != nil { err := qtx.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Collectorid: id, - Versionid: *params.ActiveVersion, + Jobid: params.JobID, + Versionid: *params.ActiveVersion, }) if err != nil { return err @@ -211,7 +209,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector if params.MinCleanVersion != nil { err = qtx.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{ - Collectorid: id, + Jobid: params.JobID, Versionid: *params.MinCleanVersion, Addedversion: latestVersion, }) @@ -222,7 +220,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector if params.MinTextVersion != nil { err = qtx.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{ - Collectorid: id, + Jobid: params.JobID, Versionid: *params.MinTextVersion, Addedversion: latestVersion, }) @@ -235,7 +233,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector removeIDs := getRemoveFields(current.Fields, params.Fields) for _, field := range removeIDs { err := qtx.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{ - Collectorid: id, + Jobid: params.JobID, Queryid: field, Removedversion: &latestVersion, }) @@ -247,7 +245,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector addIDs := getAddFields(current.Fields, params.Fields) for key, field := range addIDs { err := qtx.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Collectorid: id, + Jobid: params.JobID, Name: key, Queryid: field, Addedversion: latestVersion, diff --git a/internal/job/collector/update/update_test.go b/internal/job/collector/update/update_test.go index 12f18627..18dc8fa6 100644 --- a/internal/job/collector/update/update_test.go +++ b/internal/job/collector/update/update_test.go @@ -44,7 +44,6 @@ func TestUpdate(t *testing.T) { }) current := collector.Collector{ - ID: uuid.New(), JobID: uuid.New(), ActiveVersion: 1, LatestVersion: 4, @@ -59,18 +58,18 @@ func TestUpdate(t *testing.T) { pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(update.JobID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(current.ID), database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")), + pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). + AddRow(database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")), ) pool.ExpectBeginTx(pgx.TxOptions{}) rv := current.LatestVersion + 1 - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.ID)).WillReturnRows( + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.JobID)).WillReturnRows( pgxmock.NewRows([]string{"version"}). AddRow(rv), ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ID), int32(2)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.ID), rv, *update.MinCleanVersion). + pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), rv, *update.MinCleanVersion). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() diff --git a/internal/job/collector/update/updateprivate_test.go b/internal/job/collector/update/updateprivate_test.go index aa1cfc7e..8a04e31c 100644 --- a/internal/job/collector/update/updateprivate_test.go +++ b/internal/job/collector/update/updateprivate_test.go @@ -192,7 +192,6 @@ func TestSubmitUpdate(t *testing.T) { minCleanV := int32(2) minTextV := int32(4) current := collector.Collector{ - ID: uuid.New(), JobID: uuid.New(), ActiveVersion: 1, LatestVersion: 1, @@ -219,24 +218,24 @@ func TestSubmitUpdate(t *testing.T) { pool.ExpectBeginTx(pgx.TxOptions{}) rv := int32(2) - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.ID)).WillReturnRows( + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.JobID)).WillReturnRows( pgxmock.NewRows([]string{"version"}). AddRow(rv), ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ID), int32(2)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.ID), rv, *params.MinCleanVersion). + pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), rv, *params.MinCleanVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(current.ID), rv, *params.MinTextVersion). + pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), rv, *params.MinTextVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: RemoveCollectorQuery :exec").WithArgs(&rv, database.MustToDBUUID(current.Fields["original_key"]), database.MustToDBUUID(current.ID)). + pool.ExpectExec("name: RemoveCollectorQuery :exec").WithArgs(&rv, database.MustToDBUUID(current.Fields["original_key"]), database.MustToDBUUID(current.JobID)). WillReturnResult(pgxmock.NewResult("", 1)) pool.MatchExpectationsInOrder(false) - pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ID), "example_key", (*params.Fields)["example_key"], int32(2)). + pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.JobID), "example_key", (*params.Fields)["example_key"], int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ID), "second_key", (*params.Fields)["second_key"], int32(2)). + pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.JobID), "second_key", (*params.Fields)["second_key"], int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ID), "changed_key", (*params.Fields)["changed_key"], int32(2)). + pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.JobID), "changed_key", (*params.Fields)["changed_key"], int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() @@ -261,7 +260,6 @@ func TestSubmitUpdate(t *testing.T) { } current := collector.Collector{ - ID: uuid.New(), JobID: uuid.New(), ActiveVersion: 1, LatestVersion: 1, @@ -276,7 +274,7 @@ func TestSubmitUpdate(t *testing.T) { } pool.ExpectBeginTx(pgx.TxOptions{}) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ID), int32(2)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() diff --git a/internal/job/create.go b/internal/job/create.go index 97d1b56b..88d0b7d8 100644 --- a/internal/job/create.go +++ b/internal/job/create.go @@ -16,7 +16,7 @@ func (s *Service) Create(ctx context.Context, clientID uuid.UUID) (uuid.UUID, er id := database.MustToUUID(did) - _, err = s.svc.Collector.Create(ctx, &collector.CreateParams{ + err = s.svc.Collector.Create(ctx, &collector.CreateParams{ JobID: id, }) if err != nil { diff --git a/internal/job/create_test.go b/internal/job/create_test.go index b0432206..50d174c7 100644 --- a/internal/job/create_test.go +++ b/internal/job/create_test.go @@ -34,22 +34,17 @@ func TestCreate(t *testing.T) { ID: uuid.New(), ClientID: uuid.New(), } - collId := uuid.New() pool.ExpectQuery("name: CreateJob :one").WithArgs(database.MustToDBUUID(job.ClientID)).WillReturnRows( pgxmock.NewRows([]string{"id"}). AddRow(database.MustToDBUUID(job.ID)), ) pool.ExpectBeginTx(pgx.TxOptions{}) - pool.ExpectQuery("name: CreateCollector :one").WithArgs(database.MustToDBUUID(job.ID)).WillReturnRows( - pgxmock.NewRows([]string{"id"}). - AddRow(database.MustToDBUUID(collId)), - ) - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(collId)).WillReturnRows( + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(job.ID)).WillReturnRows( pgxmock.NewRows([]string{"version"}). AddRow(int32(1)), ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(collId), int32(1)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(job.ID), int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() diff --git a/internal/job/get.go b/internal/job/get.go index 51b28311..fb248962 100644 --- a/internal/job/get.go +++ b/internal/job/get.go @@ -29,3 +29,25 @@ func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Job, error) { CanSync: canSync, }, nil } + +type JobWithStatus struct { + Job *Job + Status Status +} + +func (s *Service) GetWithStatus(ctx context.Context, id uuid.UUID) (*JobWithStatus, error) { + job, err := s.Get(ctx, id) + if err != nil { + return nil, err + } + + status, err := s.GetStatus(ctx, id) + if err != nil { + return nil, err + } + + return &JobWithStatus{ + Job: job, + Status: *status, + }, nil +} diff --git a/internal/job/get_test.go b/internal/job/get_test.go index f7327922..e92daee1 100644 --- a/internal/job/get_test.go +++ b/internal/job/get_test.go @@ -73,3 +73,52 @@ func TestGet(t *testing.T) { CanSync: true, }, ajob) } + +func TestGetWithStatus(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := job.New(cfg, &job.Services{ + Collector: collector.New(cfg, &collector.Services{}), + Client: client.New(cfg), + }) + + j := job.Job{ + ID: uuid.New(), + ClientID: uuid.New(), + CanSync: true, + } + + pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows( + pgxmock.NewRows([]string{"id", "clientId", "canSync"}). + AddRow(database.MustToDBUUID(j.ID), database.MustToDBUUID(j.ClientID), j.CanSync), + ) + pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)). + WillReturnRows( + pgxmock.NewRows([]string{"id", "name", "canSync"}). + AddRow(database.MustToDBUUID(j.ClientID), "client_name", false), + ) + pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows( + pgxmock.NewRows([]string{"issynced"}). + AddRow(true), + ) + + ajob, err := svc.GetWithStatus(ctx, j.ID) + assert.NoError(t, err) + jStatus := job.IN_SYNC + assert.EqualExportedValues(t, &job.JobWithStatus{ + Job: &job.Job{ + ID: j.ID, + ClientID: j.ClientID, + CanSync: false, + }, + Status: jStatus, + }, ajob) +} diff --git a/internal/job/status.go b/internal/job/status.go new file mode 100644 index 00000000..f815afa3 --- /dev/null +++ b/internal/job/status.go @@ -0,0 +1,30 @@ +package job + +import ( + "context" + "queryorchestration/internal/database" + + "github.com/google/uuid" +) + +type Status string + +const ( + NOT_SYNCED Status = "not_synced" + IN_SYNC Status = "in_sync" +) + +func (s *Service) GetStatus(ctx context.Context, id uuid.UUID) (*Status, error) { + issynced, err := s.cfg.GetDBQueries().IsJobSynced(ctx, database.MustToDBUUID(id)) + if err != nil { + return nil, err + } + + status := NOT_SYNCED + switch issynced { + case true: + status = IN_SYNC + } + + return &status, nil +} diff --git a/internal/job/status_test.go b/internal/job/status_test.go new file mode 100644 index 00000000..ad2b36a2 --- /dev/null +++ b/internal/job/status_test.go @@ -0,0 +1,88 @@ +package job_test + +import ( + "context" + "errors" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + "queryorchestration/internal/job" + "queryorchestration/internal/serviceconfig" + "testing" + + "github.com/google/uuid" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" +) + +func TestGetStatus(t *testing.T) { + t.Run("is synced", func(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := job.New(cfg, &job.Services{}) + + jobId := uuid.New() + + pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(jobId)).WillReturnRows( + pgxmock.NewRows([]string{"issynced"}). + AddRow(true), + ) + + issynced, err := svc.GetStatus(ctx, jobId) + assert.NoError(t, err) + assert.Equal(t, job.IN_SYNC, *issynced) + }) + t.Run("not synced", func(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := job.New(cfg, &job.Services{}) + + jobId := uuid.New() + + pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(jobId)).WillReturnRows( + pgxmock.NewRows([]string{"issynced"}). + AddRow(false), + ) + + issynced, err := svc.GetStatus(ctx, jobId) + assert.NoError(t, err) + assert.Equal(t, job.NOT_SYNCED, *issynced) + }) + t.Run("db error", func(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := job.New(cfg, &job.Services{}) + + jobId := uuid.New() + + errStr := "db fail" + pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(jobId)). + WillReturnError(errors.New(errStr)) + + _, err = svc.GetStatus(ctx, jobId) + assert.EqualError(t, err, errStr) + }) +} diff --git a/internal/query/result/get.go b/internal/query/result/get.go index 0585dcb1..f677364b 100644 --- a/internal/query/result/get.go +++ b/internal/query/result/get.go @@ -2,6 +2,7 @@ package result import ( "context" + "errors" "fmt" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" @@ -13,27 +14,27 @@ import ( ) type GetValueWithVersionParams struct { - Type resultprocessor.Type - QueryID uuid.UUID - DocumentID uuid.UUID - QueryVersion int32 - MinCleanVersion int32 - MinTextVersion int32 + Type resultprocessor.Type + QueryID uuid.UUID + DocumentID uuid.UUID + QueryVersion int32 } func (s *Service) GetValueWithVersion(ctx context.Context, params *GetValueWithVersionParams) (resultprocessor.Value, error) { res, err := s.cfg.GetDBQueries().GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{ Queryid: database.MustToDBUUID(params.QueryID), - Queryversion: params.QueryVersion, + Queryversion: ¶ms.QueryVersion, Documentid: database.MustToDBUUID(params.DocumentID), - Cleanversion: params.MinCleanVersion, - Textversion: params.MinTextVersion, }) if err != nil { return nil, err } - return getValueByType(params.Type, res.Value) + if res.Value == nil { + return nil, errors.New("no value found") + } + + return getValueByType(params.Type, *res.Value) } func getValueByType(t resultprocessor.Type, value string) (resultprocessor.Value, error) { diff --git a/internal/query/result/get_test.go b/internal/query/result/get_test.go index 101080a6..d03ca62e 100644 --- a/internal/query/result/get_test.go +++ b/internal/query/result/get_test.go @@ -42,23 +42,22 @@ func TestGetValueWithVersion(t *testing.T) { svc := New(cfg, &Services{}) params := &GetValueWithVersionParams{ - Type: resultprocessor.TypeJsonExtractor, - QueryID: uuid.New(), - DocumentID: uuid.New(), - QueryVersion: 1, - MinCleanVersion: 2, - MinTextVersion: 3, + Type: resultprocessor.TypeJsonExtractor, + QueryID: uuid.New(), + DocumentID: uuid.New(), + QueryVersion: 1, } - pool.ExpectQuery("name: GetResultValueWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion, database.MustToDBUUID(params.DocumentID), params.MinCleanVersion, params.MinTextVersion). + value := "exaple_value" + pool.ExpectQuery("name: GetResultValueWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), ¶ms.QueryVersion, database.MustToDBUUID(params.DocumentID)). WillReturnRows( pgxmock.NewRows([]string{"id", "value"}). - AddRow(pgtype.UUID{}, "example_value"), + AddRow(pgtype.UUID{}, &value), ) val, err := svc.GetValueWithVersion(ctx, params) assert.NoError(t, err) - v := jsonextractor.NewResult("example_value") + v := jsonextractor.NewResult(value) assert.Equal(t, v, val) - assert.Equal(t, "example_value", v.GetStoreValue()) + assert.Equal(t, value, v.GetStoreValue()) } diff --git a/internal/query/result/process.go b/internal/query/result/process.go index 82e2d576..e992d3f0 100644 --- a/internal/query/result/process.go +++ b/internal/query/result/process.go @@ -71,7 +71,7 @@ func (s *Service) listRequiredValues(ctx context.Context, p *Process, query *res qResults, err := s.cfg.GetDBQueries().ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ Queryid: database.MustToDBUUID(p.QueryID), Documentid: database.MustToDBUUID(p.DocumentID), - Version: p.QueryVersion, + Version: &p.QueryVersion, }) if err != nil && !errors.Is(err, sql.ErrNoRows) { return nil, err @@ -92,7 +92,7 @@ func parseQueryRequirementValueArray(v []*repository.ListQueryRequirementValuesR return nil, err } - cleanValue, err := getValueByType(qType, r.Value) + cleanValue, err := getValueByType(qType, *r.Value) if err != nil { return nil, err } diff --git a/internal/query/result/process_test.go b/internal/query/result/process_test.go index 6dd1f4b1..530b2184 100644 --- a/internal/query/result/process_test.go +++ b/internal/query/result/process_test.go @@ -11,7 +11,6 @@ import ( "testing" "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) @@ -51,14 +50,15 @@ func TestProcess(t *testing.T) { pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)), ) - pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version). + strVal := `{"examplekey":"example_value"}` + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)). WillReturnRows( - pgxmock.NewRows([]string{"queryId", "type", "value"}). - AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeContextFull, "{\"examplekey\":\"example_value\"}"), + pgxmock.NewRows([]string{"id", "queryId", "type", "value"}). + AddRow(database.MustToDBUUID(uuid.New()), database.MustToDBUUID(query.ID), repository.QuerytypeContextFull, &strVal), ) - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(qcfg)), + pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows( + pgxmock.NewRows([]string{"config"}). + AddRow([]byte(qcfg)), ) val, err := svc.Process(ctx, ¶ms) @@ -109,16 +109,18 @@ func TestListRequiredValue(t *testing.T) { QueryVersion: query.Version, } - pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version). + strVal := "axe_value" + + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)). WillReturnRows( - pgxmock.NewRows([]string{"queryId", "type", "value"}). - AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeJsonExtractor, "example_value"), + pgxmock.NewRows([]string{"id", "queryId", "type", "value"}). + AddRow(database.MustToDBUUID(uuid.New()), database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeJsonExtractor, &strVal), ) pr, err = svc.listRequiredValues(ctx, ¶ms, query) assert.NoError(t, err) assert.ElementsMatch(t, []resultprocessor.Value{ - jsonextractor.NewResult("example_value"), + jsonextractor.NewResult(strVal), }, pr) } @@ -138,10 +140,11 @@ func TestGetProcessor(t *testing.T) { } func TestParseQueryRequirementValueArray(t *testing.T) { + exval := "exampleval" in := []*repository.ListQueryRequirementValuesRow{ { Queryid: database.MustToDBUUID(uuid.New()), - Value: "example_value", + Value: &exval, Type: repository.QuerytypeJsonExtractor, }, } @@ -149,6 +152,6 @@ func TestParseQueryRequirementValueArray(t *testing.T) { out, err := parseQueryRequirementValueArray(in) assert.NoError(t, err) assert.ElementsMatch(t, []resultprocessor.Value{ - jsonextractor.NewResult("example_value"), + jsonextractor.NewResult(exval), }, out) } diff --git a/internal/query/result/processor/parse_test.go b/internal/query/result/processor/parse_test.go index 250ab49b..cb1ba2dc 100644 --- a/internal/query/result/processor/parse_test.go +++ b/internal/query/result/processor/parse_test.go @@ -13,7 +13,7 @@ import ( func TestParseDBCollectorQuery(t *testing.T) { dbResult := repository.Collectorquerydependencytree{ - Collectorid: pgtype.UUID{}, + Jobid: pgtype.UUID{}, Queryid: pgtype.UUID{}, Requiredids: []pgtype.UUID{}, Type: repository.QuerytypeJsonExtractor, diff --git a/internal/query/result/set/set.go b/internal/query/result/set/set.go index caef1d77..a15c70fe 100644 --- a/internal/query/result/set/set.go +++ b/internal/query/result/set/set.go @@ -2,6 +2,8 @@ package resultset import ( "context" + "database/sql" + "errors" "log/slog" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" @@ -17,11 +19,6 @@ type Set struct { func (s *Service) Set(ctx context.Context, params *Set) error { dbid := database.MustToDBUUID(params.DocumentID) - cleanVersion, err := s.cfg.GetDBQueries().GetDocumentCleanEntry(ctx, dbid) - if err != nil { - return err - } - textVersion, err := s.cfg.GetDBQueries().GetDocumentTextEntry(ctx, dbid) if err != nil { return err @@ -41,13 +38,37 @@ func (s *Service) Set(ctx context.Context, params *Set) error { return err } - err = s.cfg.GetDBQueries().SetResult(ctx, &repository.SetResultParams{ - Queryid: database.MustToDBUUID(params.QueryID), - Documentid: database.MustToDBUUID(params.DocumentID), - Value: value.GetStoreValue(), - Cleanversion: cleanVersion.Version, - Textversion: textVersion.Version, - Queryversion: query.ActiveVersion, + err = s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error { + id, err := q.AddResult(ctx, &repository.AddResultParams{ + Queryid: database.MustToDBUUID(params.QueryID), + Value: value.GetStoreValue(), + Queryversion: query.ActiveVersion, + Textentryid: textVersion.ID, + }) + if err != nil { + return err + } + + qResults, err := q.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ + Queryid: database.MustToDBUUID(params.QueryID), + Documentid: database.MustToDBUUID(params.DocumentID), + Version: &query.ActiveVersion, + }) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + + for _, result := range qResults { + err = q.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: id, + Requiredresultid: result.ID, + }) + if err != nil { + return err + } + } + + return nil }) if err != nil { return err @@ -60,8 +81,8 @@ func (s *Service) Set(ctx context.Context, params *Set) error { func (s *Service) informQueryDependents(ctx context.Context, params *Set) error { ids, err := s.cfg.GetDBQueries().ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - ID: database.MustToDBUUID(params.DocumentID), - Requiredids: database.MustToDBUUID(params.QueryID), + Documentid: database.MustToDBUUID(params.DocumentID), + Queryid: database.MustToDBUUID(params.QueryID), }) if err != nil { return err diff --git a/internal/query/result/set/set_test.go b/internal/query/result/set/set_test.go index cd32c40f..b148bd30 100644 --- a/internal/query/result/set/set_test.go +++ b/internal/query/result/set/set_test.go @@ -16,7 +16,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -65,15 +64,11 @@ func TestSet(t *testing.T) { QueryID: query.ID, } - pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)). - WillReturnRows( - pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). - AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)), - ) + textEntryId := uuid.New() pool.ExpectQuery("name: GetDocumentTextEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)). WillReturnRows( - pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). - AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)), + pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "cleanEntryId"}). + AddRow(database.MustToDBUUID(textEntryId), database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1), database.MustToDBUUID(uuid.New())), ) pool.ExpectQuery("name: GetQuery :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows( pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). @@ -83,17 +78,32 @@ func TestSet(t *testing.T) { pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)), ) - pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version). + requiredResultId := uuid.New() + strVal := "{\"examplekey\":\"example_value\"}" + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)). WillReturnRows( - pgxmock.NewRows([]string{"queryId", "type", "value"}). - AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, "{\"examplekey\":\"example_value\"}"), + pgxmock.NewRows([]string{"id", "queryId", "type", "value"}). + AddRow(database.MustToDBUUID(requiredResultId), database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, &strVal), ) - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(qcfg)), + pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows( + pgxmock.NewRows([]string{"config"}). + AddRow([]byte(qcfg)), ) - pool.ExpectExec("name: SetResult :exec").WithArgs(database.MustToDBUUID(query.ID), database.MustToDBUUID(params.DocumentID), pgxmock.AnyArg(), int32(1), int32(1), query.Version). + pool.ExpectBegin() + resultId := uuid.New() + pool.ExpectQuery("name: AddResult :one").WithArgs(database.MustToDBUUID(query.ID), pgxmock.AnyArg(), database.MustToDBUUID(textEntryId), query.Version). + WillReturnRows( + pgxmock.NewRows([]string{"id"}). + AddRow(database.MustToDBUUID(resultId)), + ) + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)). + WillReturnRows( + pgxmock.NewRows([]string{"id", "queryId", "type", "value"}). + AddRow(database.MustToDBUUID(requiredResultId), database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, &strVal), + ) + pool.ExpectExec("name: AddResultDependency :exec").WithArgs(database.MustToDBUUID(resultId), database.MustToDBUUID(requiredResultId)). WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectCommit() pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(database.MustToDBUUID(params.QueryID), database.MustToDBUUID(params.DocumentID)). WillReturnRows( pgxmock.NewRows([]string{"queryId"}). diff --git a/internal/query/test/test.go b/internal/query/test/test.go index 0ac6990c..ee44de19 100644 --- a/internal/query/test/test.go +++ b/internal/query/test/test.go @@ -6,22 +6,10 @@ import ( ) func (s *Service) Test(ctx context.Context, filters result.Process) (string, error) { - doc, err := s.svc.Document.Get(ctx, filters.DocumentID) - if err != nil { - return "", err - } - - col, err := s.svc.Collector.GetByJobID(ctx, doc.JobID) - if err != nil { - return "", err - } - res, err := s.svc.Result.GetValueWithVersion(ctx, &result.GetValueWithVersionParams{ - QueryID: filters.QueryID, - DocumentID: filters.DocumentID, - QueryVersion: filters.QueryVersion, - MinCleanVersion: col.MinCleanVersion, - MinTextVersion: col.MinTextVersion, + QueryID: filters.QueryID, + DocumentID: filters.DocumentID, + QueryVersion: filters.QueryVersion, }) if err == nil { return res.GetValue(ctx) diff --git a/internal/query/test/test_test.go b/internal/query/test/test_test.go index 09cad5c5..0082b133 100644 --- a/internal/query/test/test_test.go +++ b/internal/query/test/test_test.go @@ -43,14 +43,8 @@ func TestTest(t *testing.T) { }), }) - coll := collector.Collector{ - ID: uuid.New(), - JobID: uuid.New(), - } doc := document.Document{ - ID: coll.JobID, - JobID: coll.JobID, - Hash: "example_hash", + ID: uuid.New(), } params := &result.Process{ QueryID: uuid.New(), @@ -59,28 +53,19 @@ func TestTest(t *testing.T) { } reqID := database.MustToDBUUID(uuid.New()) - pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "hash"}). - AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash), - ) - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(doc.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")), - ) pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), ¶ms.QueryVersion).WillReturnRows( pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). AddRow(database.MustToDBUUID(params.QueryID), repository.QuerytypeJsonExtractor, int32(1), params.QueryVersion+1, []byte("{\"path\":\"oldkey\"}"), []pgtype.UUID{reqID}), ) - pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(params.QueryID), params.QueryVersion). + strVal := "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}" + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.QueryID), ¶ms.QueryVersion, database.MustToDBUUID(params.DocumentID)). WillReturnRows( - pgxmock.NewRows([]string{"queryId", "type", "value"}). - AddRow(reqID, repository.QuerytypeContextFull, "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}"), + pgxmock.NewRows([]string{"id", "queryId", "type", "value"}). + AddRow(database.MustToDBUUID(uuid.New()), reqID, repository.QuerytypeContextFull, &strVal), ) - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion).WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte("{\"path\":\"oldkey\"}")), + pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID)).WillReturnRows( + pgxmock.NewRows([]string{"config"}). + AddRow([]byte("{\"path\":\"oldkey\"}")), ) result, err := svc.Test(ctx, *params) diff --git a/internal/query/types/jsonExtractor/process_test.go b/internal/query/types/jsonExtractor/process_test.go index 9d9fe265..0692c46c 100644 --- a/internal/query/types/jsonExtractor/process_test.go +++ b/internal/query/types/jsonExtractor/process_test.go @@ -12,7 +12,6 @@ import ( "testing" "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) @@ -44,10 +43,10 @@ func TestJSONProcess(t *testing.T) { config := "{\"path\":\"key\"}" - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version). + pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(config)), + pgxmock.NewRows([]string{"config"}). + AddRow([]byte(config)), ) value, err := extractor.Process(ctx, query, values) @@ -59,10 +58,10 @@ func TestJSONProcess(t *testing.T) { values = []resultprocessor.Value{ contextfull.NewResult(jsonString), } - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version). + pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(config)), + pgxmock.NewRows([]string{"config"}). + AddRow([]byte(config)), ) value, err = extractor.Process(ctx, query, values) @@ -74,10 +73,10 @@ func TestJSONProcess(t *testing.T) { values = []resultprocessor.Value{ contextfull.NewResult(jsonString), } - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version). + pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(config)), + pgxmock.NewRows([]string{"config"}). + AddRow([]byte(config)), ) value, err = extractor.Process(ctx, query, values) @@ -112,10 +111,10 @@ func TestJSONProcessJSON(t *testing.T) { config := "{\"path\":\"invalid_key\"}" - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version). + pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(config)), + pgxmock.NewRows([]string{"config"}). + AddRow([]byte(config)), ) value, err := extractor.Process(ctx, query, values) @@ -124,10 +123,10 @@ func TestJSONProcessJSON(t *testing.T) { config = "" - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version). + pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(config)), + pgxmock.NewRows([]string{"config"}). + AddRow([]byte(config)), ) value, err = extractor.Process(ctx, query, values) @@ -136,10 +135,10 @@ func TestJSONProcessJSON(t *testing.T) { config = "{\"path\":\"\"}" - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version). + pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(config)), + pgxmock.NewRows([]string{"config"}). + AddRow([]byte(config)), ) value, err = extractor.Process(ctx, query, values) @@ -148,19 +147,19 @@ func TestJSONProcessJSON(t *testing.T) { config = "{\"path\":}" - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version). + pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(config)), + pgxmock.NewRows([]string{"config"}). + AddRow([]byte(config)), ) value, err = extractor.Process(ctx, query, values) assert.EqualError(t, err, "invalid character '}' looking for beginning of value") assert.Empty(t, value) - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version). + pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}), + pgxmock.NewRows([]string{"config"}), ) value, err = extractor.Process(ctx, query, values) diff --git a/internal/query/types/jsonExtractor/service.go b/internal/query/types/jsonExtractor/service.go index 517dd2bd..ef56aabe 100644 --- a/internal/query/types/jsonExtractor/service.go +++ b/internal/query/types/jsonExtractor/service.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" "queryorchestration/internal/serviceconfig" @@ -34,16 +33,13 @@ func (e *Extractor) Process(ctx context.Context, query *resultprocessor.Query, v return "", err } - byteConfig, err := e.cfg.GetDBQueries().GetQueryConfig(ctx, &repository.GetQueryConfigParams{ - Queryid: database.MustToDBUUID(query.ID), - Addedversion: query.Version, - }) + byteConfig, err := e.cfg.GetDBQueries().GetActiveQueryConfig(ctx, database.MustToDBUUID(query.ID)) if err != nil { return "", err } var config Config - err = json.Unmarshal(byteConfig.Config, &config) + err = json.Unmarshal(byteConfig, &config) if err != nil { return "", err } diff --git a/internal/query/update/update.go b/internal/query/update/update.go index e6a8309d..998cca5d 100644 --- a/internal/query/update/update.go +++ b/internal/query/update/update.go @@ -66,8 +66,8 @@ func (s *Service) normalizeUpdateRequiredQueryIDs(ctx context.Context, current * } createsloop, err := s.cfg.GetDBQueries().IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{ - Requiredqueryid: database.MustToDBUUID(current.ID), - ID: database.MustToDBUUIDArray(*entity.GetRequiredQueryIDs()), + Queryid: database.MustToDBUUID(current.ID), + Requiredqueryids: database.MustToDBUUIDArray(*entity.GetRequiredQueryIDs()), }) if err != nil { return err diff --git a/pkg/queryService/api.gen.go b/pkg/queryService/api.gen.go index 19b59d02..a05dd30c 100644 --- a/pkg/queryService/api.gen.go +++ b/pkg/queryService/api.gen.go @@ -36,6 +36,12 @@ const ( RightClosedInterval FieldFilterCondition = "right_closed_interval" ) +// Defines values for JobStatus. +const ( + INSYNC JobStatus = "IN_SYNC" + NOTSYNCED JobStatus = "NOT_SYNCED" +) + // Defines values for QueryType. const ( CONTEXTFULL QueryType = "CONTEXT_FULL" @@ -121,6 +127,9 @@ type Job struct { // Id The job id Id openapi_types.UUID `json:"id"` + + // Status Specifies the status of a job. + Status JobStatus `json:"status"` } // JobClient defines model for JobClient. @@ -186,6 +195,9 @@ type JobCreate struct { ClientId openapi_types.UUID `json:"client_id"` } +// JobStatus Specifies the status of a job. +type JobStatus string + // JobUpdate defines model for JobUpdate. type JobUpdate struct { // CanSync Specifies whether the job is actively syncing diff --git a/serviceAPIs/queryService.yaml b/serviceAPIs/queryService.yaml index 68fccc17..f14faa98 100644 --- a/serviceAPIs/queryService.yaml +++ b/serviceAPIs/queryService.yaml @@ -32,15 +32,15 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ClientCreate' + $ref: "#/components/schemas/ClientCreate" responses: - '201': + "201": description: Client created successfully. content: application/json: schema: - $ref: '#/components/schemas/IdMessage' - '400': + $ref: "#/components/schemas/IdMessage" + "400": description: Invalid request body. /client/{id}: @@ -59,15 +59,15 @@ paths: summary: Get a client by ID description: Retrieves a specific client by its ID. responses: - '200': + "200": description: Client details. content: application/json: schema: - $ref: '#/components/schemas/JobClient' - '400': + $ref: "#/components/schemas/JobClient" + "400": description: Invalid request parameters. - '404': + "404": description: Client not found. patch: operationId: updateClient @@ -80,13 +80,13 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ClientUpdate' + $ref: "#/components/schemas/ClientUpdate" responses: - '200': + "200": description: Client updated successfully. - '400': + "400": description: Invalid request body. - '404': + "404": description: Client not found /query: @@ -97,15 +97,15 @@ paths: summary: List queries description: Retrieves a list of queries based on filter criteria. responses: - '200': + "200": description: A list of queries. content: application/json: schema: - $ref: '#/components/schemas/ListQueries' - '400': + $ref: "#/components/schemas/ListQueries" + "400": description: Invalid request parameters. - '404': + "404": description: Query not found. post: operationId: createQuery @@ -118,15 +118,15 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/QueryCreate' + $ref: "#/components/schemas/QueryCreate" responses: - '201': + "201": description: Query created successfully. content: application/json: schema: - $ref: '#/components/schemas/IdMessage' - '400': + $ref: "#/components/schemas/IdMessage" + "400": description: Invalid request body. /query/{id}: @@ -145,15 +145,15 @@ paths: summary: Get a query by ID description: Retrieves a specific query by its ID. responses: - '200': + "200": description: Query details. content: application/json: schema: - $ref: '#/components/schemas/Query' - '400': + $ref: "#/components/schemas/Query" + "400": description: Invalid request parameters. - '404': + "404": description: Query not found. patch: operationId: updateQuery @@ -166,13 +166,13 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/QueryUpdate' + $ref: "#/components/schemas/QueryUpdate" responses: - '200': + "200": description: Query updated successfully. - '400': + "400": description: Invalid request body. - '404': + "404": description: Query not found. /query/{id}/test: @@ -195,15 +195,15 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/QueryTestRequest' + $ref: "#/components/schemas/QueryTestRequest" responses: - '200': + "200": description: Test result. content: application/json: schema: - $ref: '#/components/schemas/QueryTestResponse' - '400': + $ref: "#/components/schemas/QueryTestResponse" + "400": description: Invalid request body. /job: @@ -218,15 +218,15 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/JobCreate' + $ref: "#/components/schemas/JobCreate" responses: - '201': + "201": description: Job created successfully. content: application/json: schema: - $ref: '#/components/schemas/IdMessage' - '400': + $ref: "#/components/schemas/IdMessage" + "400": description: Invalid request body. /job/{id}: @@ -245,15 +245,15 @@ paths: summary: Get a job by ID description: Retrieves a specific job by its ID. responses: - '200': + "200": description: Job details. content: application/json: schema: - $ref: '#/components/schemas/Job' - '400': + $ref: "#/components/schemas/Job" + "400": description: Invalid request parameters. - '404': + "404": description: Job not found. patch: operationId: updateJob @@ -266,13 +266,13 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/JobUpdate' + $ref: "#/components/schemas/JobUpdate" responses: - '204': + "204": description: Job updated successfully. - '400': + "400": description: Invalid request body. - '404': + "404": description: Job not found. /job/{id}/collector: @@ -291,13 +291,13 @@ paths: summary: Get a job collector by ID description: Retrieves a specific job collector by its ID. responses: - '200': + "200": description: Job collector details. content: application/json: schema: - $ref: '#/components/schemas/JobCollector' - '404': + $ref: "#/components/schemas/JobCollector" + "404": description: Job collector not found. patch: operationId: updateJobCollectorByJobId @@ -310,13 +310,13 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/JobCollectorUpdate' + $ref: "#/components/schemas/JobCollectorUpdate" responses: - '204': + "204": description: Job collector updated successfully. - '400': + "400": description: Invalid request body. - '404': + "404": description: Job collector not found. /job/export: @@ -331,17 +331,17 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ExportTrigger' + $ref: "#/components/schemas/ExportTrigger" responses: - '201': + "201": description: Export triggered successfully. content: application/json: schema: - $ref: '#/components/schemas/IdMessage' - '400': + $ref: "#/components/schemas/IdMessage" + "400": description: Invalid request body. - '404': + "404": description: Job not found. /job/{id}/export: @@ -360,16 +360,15 @@ paths: summary: Check export state. description: Checks the current state of an export. responses: - '200': + "200": description: Export has been completed. content: application/json: schema: - $ref: '#/components/schemas/ExportDetails' - '404': + $ref: "#/components/schemas/ExportDetails" + "404": description: Export job not found. - components: schemas: QueryType: @@ -387,7 +386,7 @@ components: format: uuid description: Unique identifier for the query. type: - $ref: '#/components/schemas/QueryType' + $ref: "#/components/schemas/QueryType" active_version: type: integer format: int32 @@ -417,7 +416,7 @@ components: queries: type: array items: - $ref: '#/components/schemas/Query' + $ref: "#/components/schemas/Query" description: List of queries. required: - queries @@ -426,7 +425,7 @@ components: type: object properties: type: - $ref: '#/components/schemas/QueryType' + $ref: "#/components/schemas/QueryType" config: type: string description: Configuration for the new query. @@ -498,6 +497,13 @@ components: - name - can_sync + JobStatus: + type: string + enum: + - IN_SYNC + - NOT_SYNCED + description: Specifies the status of a job. + Job: type: object properties: @@ -512,10 +518,13 @@ components: can_sync: type: boolean description: Specifies whether the job is actively syncing + status: + $ref: "#/components/schemas/JobStatus" required: - id - client_id - can_sync + - status JobCreate: type: object @@ -590,7 +599,7 @@ components: type: array description: The fields in the job collector. items: - $ref: '#/components/schemas/JobCollectorField' + $ref: "#/components/schemas/JobCollectorField" description: JobCollector model. required: - job_id @@ -619,7 +628,7 @@ components: type: array description: The fields in the job collector. items: - $ref: '#/components/schemas/JobCollectorField' + $ref: "#/components/schemas/JobCollectorField" description: Payload for updating a JobCollector. JobCollectorField: @@ -658,7 +667,7 @@ components: type: array description: Filter the scope based on field output values. items: - $ref: '#/components/schemas/FieldFilter' + $ref: "#/components/schemas/FieldFilter" description: Payload for triggering an export. required: - job_id @@ -711,7 +720,7 @@ components: format: uuid description: The job id relative to the export. status: - $ref: '#/components/schemas/ExportStatus' + $ref: "#/components/schemas/ExportStatus" output_location: type: string description: The location in which the export zip file will be found. diff --git a/test/queryService/job_test.go b/test/queryService/job_test.go index bb5ed7d4..e17b3159 100644 --- a/test/queryService/job_test.go +++ b/test/queryService/job_test.go @@ -42,6 +42,7 @@ func TestJob(t *testing.T) { assert.Equal(t, id, jobRes.JSON200.Id) assert.Equal(t, clientRes.JSON201.Id, jobRes.JSON200.ClientId) assert.False(t, jobRes.JSON200.CanSync) + assert.Equal(t, queryservice.INSYNC, jobRes.JSON200.Status) updateCanSync := !jobRes.JSON200.CanSync updateRes, err := client.UpdateJobWithResponse(ctx, id, queryservice.JobUpdate{ @@ -55,4 +56,5 @@ func TestJob(t *testing.T) { assert.Equal(t, id, jobRes.JSON200.Id) assert.Equal(t, clientRes.JSON201.Id, jobRes.JSON200.ClientId) assert.False(t, jobRes.JSON200.CanSync) + assert.Equal(t, queryservice.INSYNC, jobRes.JSON200.Status) }