jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/CollectorBuildVersion (pull request #100)

Browse Source 
Collector Build Version

* lint

* fixtests
This commit is contained in:
Michael McGuinness
2025-03-11 18:15:49 +00:00
parent 6648cdf1cb
commit bbd86fb4ea
51 changed files with 305 additions and 488 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/docCleanRunner/runner_test.go
+2 -5
View File
@@ -13,7 +13,6 @@ import (
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
documentclean "queryorchestration/internal/document/clean"
cleanversion "queryorchestration/internal/document/clean/version"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documenttext"
@@ -53,9 +52,7 @@ func TestDocCleanRunner(t *testing.T) {
cfg.DocumentTextURL = "/i/am/here"
runner := doccleanrunner.New(validator.New(), &doccleanrunner.Services{
Clean: documentclean.New(cfg, &documentclean.Services{
Version: cleanversion.New(cfg),
}),
Clean: documentclean.New(cfg),
})
assert.NotNil(t, runner)
@@ -110,7 +107,7 @@ func TestDocCleanRunner(t *testing.T) {
pgxmock.NewRows([]string{"id"}).
AddRow(cleanid),
)
pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(cleanid, int32(1)).
pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(cleanid, pgxmock.AnyArg()).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
api/docTextRunner/runner_test.go
+4 -7
View File
@@ -11,7 +11,6 @@ import (
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
documenttext "queryorchestration/internal/document/text"
textversion "queryorchestration/internal/document/text/version"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/queue/querysync"
queuemock "queryorchestration/mocks/queue"
@@ -45,9 +44,7 @@ func TestDocCleanRunner(t *testing.T) {
cfg.QuerySyncURL = "/i/am/here"
runner := doctextrunner.New(validator.New(), &doctextrunner.Services{
Text: documenttext.New(cfg, &documenttext.Services{
Version: textversion.New(cfg),
}),
Text: documenttext.New(cfg),
})
assert.NotNil(t, runner)
@@ -79,13 +76,13 @@ func TestDocCleanRunner(t *testing.T) {
}
pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(database.MustToDBUUID(doc.ID)).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(cleanId, database.MustToDBUUID(doc.ID), &inloc.Bucket, &inloc.Key, int32(1), dbmimetype, repository.NullCleanfailtype{}),
AddRow(cleanId, database.MustToDBUUID(doc.ID), &inloc.Bucket, &inloc.Key, int64(1), dbmimetype, repository.NullCleanfailtype{}),
)
pool.ExpectQuery("name: GetCleanEntry :one").WithArgs(cleanId).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(cleanId, database.MustToDBUUID(doc.ID), &inloc.Bucket, &inloc.Key, int32(1), dbmimetype, repository.NullCleanfailtype{}),
AddRow(cleanId, database.MustToDBUUID(doc.ID), &inloc.Bucket, &inloc.Key, int64(1), dbmimetype, repository.NullCleanfailtype{}),
)
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(int32(1), inloc.Bucket, inloc.Key, cleanId).
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(pgxmock.AnyArg(), inloc.Bucket, inloc.Key, cleanId).
WillReturnResult(pgxmock.NewResult("", 1))
mockSQS.EXPECT().
api/queryService/api.gen.go
+49 -48
View File
@@ -97,7 +97,7 @@ type ClientUpdate struct {
}
// CodeVersion The desired code version.
type CodeVersion = int32
type CodeVersion = int64
// Collector Collector model.
type Collector struct {
@@ -641,53 +641,54 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL
// Base64 encoded, gzipped, json marshaled Swagger object
var swaggerSpec = []string{
"H4sIAAAAAAAC/8xb61PjOBL/V1S++5gHmcDw+MYCu5WrPZgD5mqv9qiUI7cT7TiWkeTM5Kbyv1/p5aeU",
"OBnC7jfAUqv71091i+8BpsuMppAKHlx9D7KQhUsQwNRvNwmBVExu5c8RcMxIJghNg6vgeQEIq69ocjsI",
"egGRf81CsQh6QRouIbgKSBT0AgavOWEQBVeC5dALOF7AMpQE/84gDq6Cvw1LDob6Kx8WB282veDuW0aZ",
"lwtQX4/CRXGw5OJfObC1j4nJLaIxEgtAr3LZ27NiT99IVhjwjKYclI4m6SpMSPQIrzlw0WbPfEdML0Az",
"Gq0HSiJDu1T0TZg+rVPsoKFlMwonHIVJQr+SdI5CLMgKEF+nmAe9QKwzKeiM0gTCNNj0LGUGoQA3dhmj",
"GTBBgKOYMoTlUkJTiWH5SW4lUXezsZh3WX8vV2pcrWp+17pSNF4KsejsD8CilGqHY8A3ASwNE6Rowbdw",
"mSWSzPX1dQkVF4yk8yrNn2i03kqXRD+GTVtSv4j3BkUvMwqiunQMBPgFfBKhyHmb5lMGmMTSCqSpcbVK",
"OlVoTpIyQ5ovJcuT++nTf+5vgl5w//Csfry7rfwyuf+lIpKbATfK11Yscz5JY8qWyh6RhqYNPS8E2g2/",
"Eb6pAkPCr4bPO0yNpMbU8pS85tC0uJPR5dnF2Rj3L+Io6p+f46g/G3/E/Q9nZx9OR7MoisbjoBdoWYOr",
"IM8VAQ+An7OoizOLRSjQMlyjGaBcbXHYLQ7TKTcxZzd8NkAd5t5tbGkE/wbGFfsuaSLgUkUI0wjQSq8c",
"VIEiqRh/KJGSapgD07STBLCgrE25+ISWNIKkDYoOqtNVydo2Ka0Em16grWG6X6CMCSTRbvu1TP+sl296",
"QRIK4OIANpckJct8OcUySQDrSqGqrgoVAd/EQSQaTliCV2DiZ9VzfAuUXlOZTh+vYes2RcVRM1eqpGw3",
"t+2ok4/Uzta+0gtUFdPBjsqypI6lyQoFmd1S+xMNzUWWCwOAJDxwhqaGffpBlFG9jRwRsNzTCYIypoSM",
"hesaG0/gqMU+heuEhpHSnQqJqoRCN34VHhwKDvbqv45ztizmlmIduHbmnkbd8EY557gFaC/Iux7w2V3G",
"aQo9632FnC9uLPOlF8rIfK1g2sZxluMv4CGgvyFMUxGSVJq5qJB1OfBu2S3LGt4v4KmS5Z0LCbrjQBd6",
"RiBNextovmqsgE2X6I0C7CSK+7NxfNE/H59e9C/D2ag/grOz2TgeXUB03qUA0zfSWxAhSfj2+GLuxYKR",
"+RwYsndGhzscUjTooDxNKA6Ft3yyX2XA/bogeKF0Yhj7H8lQTBJAX0mSyBoxpnnagI2Pr4bD71otU2nV",
"m+F3e6uaFnxvhuv1er1cRpH8KonLPw4wX/03dYHYrWDXUHsK9mqtsKV479S58NgK9E+js1H/fDTG/csL",
"gP7px1O4GF+MRvFo3N1WfNct5SmUczJLCk6kJNrR7UVL4pKAgEg1NKYZo3MGXBZGcUgSiJzXLH3wsza8",
"7UZqrFOlwdSw0bZQlcumMUlsX6pO8Gf1QV8bMc0AzUIOEaKpqRlMAbEKk1xL1ynXq4yoSbsSPUnnwCUD",
"h/BVbEZlv60tNqTR1H/TSkIukPwsk11BcOAxeCa8pAhHMWEdibXTsitRV7HzwKIrH0yTfJm2QxJNIyI6",
"lAyVg26KPbb6mR5e/2pbcQOvv6GcQ5wnNtFoI6gZV0sNdRNqRJQKx72K/AUrL9thvqkitsXTtUfEhQqK",
"k2punwDnU7EI5flz1bpj9lecUA7RVDUcVmES9AKaQVr9PYFYTNvLGJkvXH8nKU7ySPeQ9E+umDKJ/gmc",
"h3Nw9Ww4SecJIBkIfZ2x+pbPtksCqSAxAaYzZiqIWA92x9ZuDbRfCRe2YODb64XySlKUrJ1iVFHEOQKU",
"PF5ez4jLjq8RByF9/VWvaMP26tsq6TZ2duJVXRV3uoE91gWopuCQJaFzglGeEsUYfAOcuzvIhzdWaBqT",
"eScRb/TSThVtcX3+gWaKRW9a0di2rY9mvTmblyrpwOuzXOisnRWJVrejJZdXrzcFws1mmfx7znQhaZse",
"xZSlFSg0sW3jhiLjIiuEDOJq7AAoLEm3UtL+FvDn60bt9oLuq0wVBv5LDMQfz/rnH88/9i8iuOxfnp6O",
"z8LL8Xh8HnYpTDW3wIV3ZuUYCxnFIGlNdpbV1pINqNN9b5O6U7Wv9zXArp7eJOnVgQZCX87cSNirG4oZ",
"XdaAaAOgqoU2mUfgeSJqM8qCwPbkpgn6uTfWuW2WI3e2xqO21PjH08P99O6358frm+eHx6AX3DzcP9/9",
"9jz9+fOvvzpLAXXu4TOIqjX92Rnix+ODq/purfLm8CIAaouY3O6ZzrXvNCuPTiOVfaYpG3Xfiqkj93+a",
"FEVUPUMo/tADwwvgwmQPDmxFsK5ZBBEqqLnWXX+ayMLbihGMBieDE9X1yCANMxJcBePByUDewrNQLBRW",
"Q1x0JjPqimk6J3EUohS+2rndVyJ0UyRjdEUiiFCkezsDXVhrhiZRsd/0P7XpABd2jIlpKszxYZYlRDde",
"hn9wrYd9Xl2Y5KlQd+lP8efOnbjKXfnIoflk4cPJ6I15NlNzB8/6u2ExQjzHGDiP8yRZD6RKT09OfEcU",
"PA8bbyzU44l8uQxlOWoUU9OrtK9wzmV8M3NfbXnBi9xqTGX4nUQbefbc1UR9BMEIrJTBcB1OsbWa2RoR",
"wc2Tl7qV/AKiYiI1zE/eDPOyD+8HvLDjN8H4FxCFeUnxJ7dbIO7VXjH97j64XFLpa74oh8YLx41RJQ+u",
"u1SEq8FN1Yel7r2uqze/g+uarLjDdQU1ubCDo544ApkW26bTI3iUFqMaTzp60xBXp+57+FUxkTeuVXva",
"1vYwu/yntbGd6JjuVj4lcLlbwfpRPK4KTAFKVSHlvPOY7vcEwh5fsrTb8Z78ujqCC1ZHv7tdcAaqB+Mc",
"4+/yytNtj0ok1SM45VPdILbbQNMvo2ofbIdfSjQSU6CWDbLZuvoE0eWYtX7b+3hmvcW3JRkWYkjB3kgh",
"qogXtS6ivtq0ombB4ps4aVO3emrTfDy8r99T99tVIojKu5XBYcaotO22AZiJkx4/HcnJ67Mtj5cbRo0X",
"K+QH71oUl91yB4d3tcHwcTK4Aagc6lWs0cwlvbGiHM92CBR8nWL7WLNT7javP98ncTffm/ojhBHh7XO4",
"fc9a4nScyllqUat6+73mZgH4i9YdzhmzsqsGUW0EXNdcOcyGY2qs/r7C7zyLkKMZQIqKAflbXSYlPLV5",
"/GCL6+yrr/IfGpS+Xu1AZWelnNSHPtUJu5pxY0YEMBK6s7IdQx05E9tjHGq7bkrwljn4tRDP6kk1lWpq",
"6tAS0j24vTtCeip2nGxXHaYc0g96rfD2F8h8utf3Xq0gK7zHKAoHPKAPpE1lexuotIsjuZwZ6PpgPsZ1",
"tJC7dgdte9teQbHoZO/X/am4a4fmz9G9dHvrRzdr7NMVe+e0vfhiFnJAN0gr+z2aQXv501CYweIP2IIz",
"Zt+p1wXKHfUoMk/1a2Nf+K6/8GrcWICLo1tGddLqMY/GaHWfS8vJMZg101AXt5pHnidvdY1WFHdbl9wD",
"bGXtqPGkkdEox8WMCVjQC3KWBFfBQoiMXw2HYUYGZpI+wHQ5XI2UjZnTmvQerJXIvJooz5JpVVXbvPx/",
"0nr5vul1JGPbNVVKzRZOV2JlY6eg1eo2dKVV1lGGUk0HXano2rlCpV40b142/w8AAP//JXKOSfQ8AAA=",
"H4sIAAAAAAAC/8xbWXPjNhL+KyjuPuqwRr7fHNtJaStrZ23PVrayLhUFNiUkFEEDoGa0U/rvW7h4AhKl",
"sZy82SbQ6P76RDf8LcB0mdEUUsGD629BFrJwCQKY+u02IZCKyZ38OQKOGckEoWlwHbwsAGH1FU3uBkEv",
"IPKvWSgWQS9IwyUE1wGJgl7A4C0nDKLgWrAcegHHC1iGkuDfGcTBdfC3YcnBUH/lw+LgzaYX3H/NKPNy",
"AerrUbgoDpZc/CsHtvYxMblDNEZiAehNLnt/VuzpG8kKA57RlIPS0SRdhQmJnuAtBy7a7JnviOkFaEaj",
"9UBJZGiXir4N0+d1ih00tGxG4YSjMEnoF5LOUYgFWQHi6xTzoBeIdSYFnVGaQJgGm56lzCAU4MYuYzQD",
"JghwFFOGsFxKaCoxLD/JrSTqbjYW8y7rH+RKjatVzW9aV4rGayEWnf0OWJRS7XAM+CqApWGCFC34Gi6z",
"RJK5ubkpoeKCkXRepfkDjdZb6ZLo+7BpS+oX8cGg6GVGQVSXjoEAv4DPIhQ5b9N8zgCTWFqBNDWuVkmn",
"Cs1JUmZI86VkefIwff7Pw23QCx4eX9SP93eVXyYPP1VEcjPgRvnGimXOJ2lM2VLZI9LQtKHnhUC74TfC",
"N1VgSPjV8HmHqZHUmFqekrccmhZ3Mro6uzwb4/5lHEX9iwsc9Wfjc9z/dHb26XQ0i6JoPA56gZY1uA7y",
"XBHwAPg5i7o4s1iEAi3DNZoBytUWh93iMJ1yE3N2w2cD1GHu3caWRvBvYFyx75ImAi5VhDCNAK30ykEV",
"KJKK89MSKamGOTBNO0kAC8ralItPaEkjSNqg6KA6XZWsbZPSSrDpBdoapvsFyphAEu22X8v0j3r5phck",
"oQAuDmBzSVKyzJdTLJMEsK4UquqqUBHwVRxEouGEJXgFJn5WPce3QOk1len08Rq2blNUHDVzpUrKdnPb",
"jjr5SO1s7Su9QFUxHeyoLEvqWJqsUJDZLbU/0dBcZLkwAEjCA2doatinH0QZ1dvIEQHLPZ0gKGNKyFi4",
"rrHxDI5a7JdwndAwUrpTIVGVUOjWr8KDQ8HBXv3Xcc6WxdxRrAPXztzTqBveKecctwDtBXnXAz67yzhN",
"oWe9r5Dz1Y1lvvRCGZmvFUzbOM5y/Ad4COhvCNNUhCSVZi4qZF0OvFt2y7KG9w/wVMnyzoUE3XGgCz0j",
"kKa9DTRfNVbApkv0RgF2EsX92Ti+7F+MTy/7V+Fs1B/B2dlsHI8uIbroUoDpG+kdiJAkfHt8Mfdiwch8",
"DgzZO6PDHQ4pGnRQniYUh8JbPtmvMuB+WRC8UDoxjP2PZCgmCaAvJElkjRjTPG3AxsfXw+E3rZaptOrN",
"8Ju9VU0LvjfD9Xq9Xi6jSH6VxOUfB5iv/pu6QOxWsGuoPQV7tVbYUrx36lx4bAX6p9HZqH8xGuP+1SVA",
"//T8FC7Hl6NRPBp3txXfdUt5CuWczJKCEymJdnR70ZK4JCAgUg2NacbonAGXhVEckgQi5zVLH/yiDW+7",
"kRrrVGkwNWy0LVTlsmlMEtuXqhP8UX3Q10ZMM0CzkEOEaGpqBlNArMIk19J1yvUqI2rSrkRP0jlwycAh",
"fBWbUdlva4sNaTT137SSkAskP8tkVxAceAyeCS8pwlFMWEdi7bTsStRV7Dyw6MoH0yRfpu2QRNOIiA4l",
"Q+Wg22KPrX6mh9e/2lbcwOtvKOcQ54lNNNoIasbVUkPdhBoRpcJxryJ/wcrrdphvq4ht8XTtEXGhguKk",
"mtsnwPlULEJ5/ly17pj9FSeUQzRVDYdVmAS9gGaQVn9PIBbT9jJG5gvX30mKkzzSPST9kyumTKJ/Aufh",
"HFw9G07SeQJIBkJfZ6y+5bPtkkAqSEyA6YyZCiLWg92xtVsD7WfChS0Y+PZ6obySFCVrpxhVFHGOACWP",
"l9cz4rLjG8RBSF9/0yvasL35tkq6jZ2deFVXxZ1uYI91AaopOGRJ6JxglKdEMQZfAefuDvLhjRWaxmTe",
"ScRbvbRTRVtcn7+jmWLRm1Y0tm3rk1lvzualSjrw+iIXOmtnRaLV7WjJ5dXrbYFws1km/54zXUjapkcx",
"ZWkFCk1s27ihyLjICiGDuBo7AApL0q2UtL8F/Pm6Ubu9oPsqU4WB/xID8flZ/+L84rx/GcFV/+r0dHwW",
"Xo3H44uwS2GquQUuvDMrx1jIKAZJa7KzrLaWbECd7nub1J2qfb2vAXb19CZJrw40EPpy5kbCXt1QzOiy",
"BkQbAFUttMk8Ac8TUZtRFgS2JzdN0M+9sc5tsxy5szUetaXGP54fH6b3v7483dy+PD4FveD28eHl/teX",
"6Y+ff/7ZWQqocw+fQVSt6c/OEN8fH1zVd2uVN4cXAVBbxORuz3SufadZeXQaqXimKeNPjmnKRt23YurI",
"/b9MiiKqniEUf+iR4QVwYbIHB7YiWNcsgggV1Fzrbn6ZyMLbihGMBieDE9X1yCANMxJcB+PByUDewrNQ",
"LBRWQ1x0JjPqimk6J3EUohS+2LndFyJ0UyRjdEUiiFCkezsDXVhrhiZRsd/0P7XpABd2jIlpKszxYZYl",
"RDdehr9zrYd9Xl2Y5KlQd+lP8efOnbjKXfnIoflk4dPJ6J15NlNzB8/6u2ExQjzHGDiP8yRZD6RKT09O",
"fEcUPA8bbyzU44l8uQxlOWoUU9OrtK9wzmV8M3NfbXnBq9xqTGX4jUQbefbc1UR9AsEIrJTBcB1OsbWa",
"2RoRwc2Tl7qV/ASiYiI1zE/eDfOyD+8HvLDjd8H4JxCFeUnxJ3dbIO7VXjH95j64XFLpa74qh8YLx41R",
"JQ+uu1SEq8FN1Yel7r2uqzd/gOuarLjDdQU1ubCDo544ApkW26bTI3iUFqMaTzp60xBXp+57+FUxkTeu",
"VXva1vYwu/yHtbGd6JjuVj4lcLlbwfpRPK4KTAFKVSHlvPOY7vcMwh5fsrTb8Z79ujqCC1ZHv7tdcAaq",
"B+Mc4+/yytNtj0ok1SM45XPdILbbQNMvo2ofbIdfSjQSU6CWDbLZuvoE0eWYtX7bx3hmvcW3JRkWYkjB",
"3kkhqogXtS6ivtq0ombB4rs4aVO3emrTfDy8r99T99tVIojKu5XBYcaotO22AZiJkx4/HcnJ67Mtj5cb",
"Ro0XK+QHH1oUl91yB4f3tcHwcTK4Aagc6lWs0cwlvbGiHM92CBR8nWL7WLNT7javPz8mcTffm/ojhBHh",
"/XO4fc9a4nScyllqUat6+73mdgH4D607nDNmZVcNotoIuK65cpgNx9RY/X2F33kWIUczgBQVA/L3ukxK",
"eGrz+MEW19lXX+U/NCh9vdmBys5KOakPfaoTdjXjxowIYCR0Z2U7hjpyJrbHONR205TgPXPwWyGe1ZNq",
"KtXU1KElpHtwe3eE9FTsONmuOkw5pB/0VuHtL5D5dK/vo1pBVniPURQOeEAfSJvK9jZQaRdHcjkz0PXB",
"fIzraCF37Q7a9ra9gmLRyd6v+1Nx1w7Nn6N76fbWj27W2Kcr9s5pe/HFLOSAbpBW9kc0g/byp6Ewg8Xv",
"sAVnzL5XrwuUO+pRZJ7q18a+8F1/4dW4sQAXR7eM6qTVYx6N0eo+l5aTYzBrpqEubjWPPE/e6xqtKO62",
"LrkH2MraUeNJI6NRjosZE7CgF+QsCa6DhRAZvx4Ow4wMzCR9gOlyuBopGzOnNek9WiuReTVRniXTqqq2",
"efn/pPXyfdPrSMa2a6qUmi2crsTKxk5Bq9Vt6EqrrKMMpZoOulLRtXOFSr1o3rxu/h8AAP//6gjhz/Q8",
"AAA=",
}
// GetSwagger returns the content of the embedded swagger specification file
api/queryService/collector_test.go
+4 -8
View File
@@ -13,8 +13,6 @@ import (
collectorset "queryorchestration/internal/collector/set"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
cleanversion "queryorchestration/internal/document/clean/version"
textversion "queryorchestration/internal/document/text/version"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/queue/clientsync"
queuemock "queryorchestration/mocks/queue"
@@ -50,7 +48,7 @@ func TestSetCollector(t *testing.T) {
}
av := int32(2)
cv := int32(1)
cv := int64(1)
body := queryservice.CollectorSet{
ActiveVersion: &av,
MinimumCleanerVersion: &cv,
@@ -72,9 +70,7 @@ func TestSetCollector(t *testing.T) {
cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
CollectorSet: collectorset.New(cfg, &collectorset.Services{
CleanVersion: cleanversion.New(cfg),
TextVersion: textversion.New(cfg),
Collector: collector.New(cfg),
Collector: collector.New(cfg),
}),
})
@@ -144,8 +140,8 @@ func TestGetCollectorByclientId(t *testing.T) {
coll := collector.Collector{
ClientID: uuid.New(),
MinCleanVersion: int32(1),
MinTextVersion: int32(2),
MinCleanVersion: 1,
MinTextVersion: 2,
}
id := "clientid"
cmd/docCleanRunner/main.go
+1 -4
View File
@@ -14,7 +14,6 @@ import (
doccleanrunner "queryorchestration/api/docCleanRunner"
documentclean "queryorchestration/internal/document/clean"
cleanversion "queryorchestration/internal/document/clean/version"
"queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documenttext"
@@ -34,9 +33,7 @@ func main() {
cfg := &DocCleanConfig{}
cfg.ControllerFunc = func() runner.Controller {
clean := documentclean.New(cfg, &documentclean.Services{
Version: cleanversion.New(cfg),
})
clean := documentclean.New(cfg)
return doccleanrunner.New(cfg.GetValidator(), &doccleanrunner.Services{
Clean: clean,
cmd/docTextRunner/main.go
+1 -4
View File
@@ -14,7 +14,6 @@ import (
doctextrunner "queryorchestration/api/docTextRunner"
documenttext "queryorchestration/internal/document/text"
textversion "queryorchestration/internal/document/text/version"
"queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/queue/querysync"
@@ -32,9 +31,7 @@ func main() {
cfg := &DocTextConfig{}
cfg.ControllerFunc = func() runner.Controller {
text := documenttext.New(cfg, &documenttext.Services{
Version: textversion.New(cfg),
})
text := documenttext.New(cfg)
return doctextrunner.New(cfg.GetValidator(), &doctextrunner.Services{
Text: text,
cmd/metricsExample_test/main.go
+3 -3
View File
@@ -23,7 +23,7 @@ import (
"syscall"
"time"
"queryorchestration/internal/serviceconfig/observability"
"queryorchestration/internal/serviceconfig/build"
"queryorchestration/internal/serviceconfig/observability/prometheus"
)
@@ -54,9 +54,9 @@ func getMetricsConfig() prometheus.MetricsConfig {
}
func main() {
rawVersion := observability.GetVersion()
rawVersion := build.GetVersion()
fmt.Printf("Version: %s\n", rawVersion)
timeVersion := observability.GetVersionTimestamp()
timeVersion := build.GetVersionTimestamp()
fmt.Printf("Version Timestamp: %s\n", timeVersion.String())
ctx, cancel := context.WithCancel(context.Background())
cmd/queryService/main.go
+1 -7
View File
@@ -24,8 +24,6 @@ import (
"queryorchestration/internal/collector"
collectorset "queryorchestration/internal/collector/set"
"queryorchestration/internal/document"
cleanversion "queryorchestration/internal/document/clean/version"
textversion "queryorchestration/internal/document/text/version"
"queryorchestration/internal/export"
"queryorchestration/internal/query"
"queryorchestration/internal/query/result"
@@ -56,13 +54,9 @@ func main() {
res := result.New(cfg, &result.Services{
Query: que,
})
tev := textversion.New(cfg)
clv := cleanversion.New(cfg)
col := collector.New(cfg)
colupdate := collectorset.New(cfg, &collectorset.Services{
Collector: col,
CleanVersion: clv,
TextVersion: tev,
Collector: col,
})
cli := client.New(cfg)
doc := document.New(cfg)
database/migrations/00000000000004_collectors.up.sql
+2 -2
View File
@@ -33,7 +33,7 @@ CREATE TABLE collectorActiveVersions (
CREATE TABLE collectorMinCleanVersions (
id uuid primary key DEFAULT uuid_generate_v7(),
clientId uuid not null,
versionId int not null,
versionId bigint not null,
addedVersion int not null,
removedVersion int,
foreign key (clientId, addedVersion) references collectorVersions(clientId, id),
@@ -60,7 +60,7 @@ EXECUTE FUNCTION removeMinCleanVersion();
CREATE TABLE collectorMinTextVersions (
id uuid primary key DEFAULT uuid_generate_v7(),
clientId uuid not null,
versionId int not null,
versionId bigint not null,
addedVersion int not null,
removedVersion int,
foreign key (clientId, addedVersion) references collectorVersions(clientId, id),
database/migrations/00000000000005_documents.up.sql
+2 -2
View File
@@ -45,14 +45,14 @@ CREATE TABLE documentCleans (
CREATE TABLE documentCleanEntries (
id uuid primary key DEFAULT uuid_generate_v7(),
cleanId uuid not null,
version int not null,
version bigint not null,
foreign key (cleanId) references documentCleans(id)
);
CREATE TABLE documentTextExtractions (
id uuid primary key DEFAULT uuid_generate_v7(),
cleanEntryId uuid not null,
version int not null,
version bigint not null,
bucket text not null,
key text not null,
foreign key (cleanEntryId) references documentCleans(id)
internal/collector/get_test.go
+2 -2
View File
@@ -28,8 +28,8 @@ func TestGetByClientID(t *testing.T) {
svc := collector.New(cfg)
minCleanV := int32(2)
minTextV := int32(4)
minCleanV := int64(2)
minTextV := int64(4)
ogc := collector.Collector{
ClientID: uuid.New(),
MinCleanVersion: minCleanV,
internal/collector/parse_test.go
+2 -2
View File
@@ -16,8 +16,8 @@ func TestParseDBCollector(t *testing.T) {
assert.NoError(t, err)
assert.Nil(t, c)
minCleanV := int32(1)
minTextV := int32(2)
minCleanV := int64(1)
minTextV := int64(2)
ogc := Collector{
ClientID: uuid.New(),
MinCleanVersion: minCleanV,
internal/collector/service.go
+2 -2
View File
@@ -8,8 +8,8 @@ import (
type Collector struct {
ClientID uuid.UUID
MinCleanVersion int32
MinTextVersion int32
MinCleanVersion int64
MinTextVersion int64
ActiveVersion int32
LatestVersion int32
Fields map[string]uuid.UUID
internal/collector/set/service.go
+1 -5
View File
@@ -2,16 +2,12 @@ package collectorset
import (
collector "queryorchestration/internal/collector"
cleanversion "queryorchestration/internal/document/clean/version"
textversion "queryorchestration/internal/document/text/version"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/queue/clientsync"
)
type Services struct {
CleanVersion *cleanversion.Service
TextVersion *textversion.Service
Collector *collector.Service
Collector *collector.Service
}
type ConfigProvider interface {
internal/collector/set/set.go
+9 -8
View File
@@ -9,6 +9,7 @@ import (
"queryorchestration/internal/collector"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/serviceconfig/build"
"queryorchestration/internal/serviceconfig/queue"
"queryorchestration/internal/validation"
@@ -19,12 +20,12 @@ import (
func (s *Service) SetByClientExternalId(ctx context.Context, id string, params *SetParams) error {
client, err := s.cfg.GetDBQueries().GetClientByExternalId(ctx, id)
if err != nil {
return nil
return err
}
err = s.SetByClientId(ctx, database.MustToUUID(client.ID), params)
if err != nil {
return nil
return err
}
return nil
@@ -32,8 +33,8 @@ func (s *Service) SetByClientExternalId(ctx context.Context, id string, params *
type SetParams struct {
ActiveVersion *int32
MinCleanVersion *int32
MinTextVersion *int32
MinCleanVersion *int64
MinTextVersion *int64
Fields *map[string]uuid.UUID
}
@@ -77,8 +78,8 @@ func (s *Service) informSet(ctx context.Context, update *dbSetParams) error {
type dbSetParams struct {
ClientID pgtype.UUID
ActiveVersion *int32
MinCleanVersion *int32
MinTextVersion *int32
MinCleanVersion *int64
MinTextVersion *int64
Fields *map[string]pgtype.UUID
}
@@ -136,7 +137,7 @@ func (s *Service) normalizeCodeVersions(current *collector.Collector, params *Se
}
if params.MinCleanVersion != nil {
err := s.svc.CleanVersion.IsValidVersion(*params.MinCleanVersion)
err := build.IsValidUnixVersion(*params.MinCleanVersion)
if err != nil {
return err
}
@@ -147,7 +148,7 @@ func (s *Service) normalizeCodeVersions(current *collector.Collector, params *Se
}
if params.MinTextVersion != nil {
err := s.svc.TextVersion.IsValidVersion(*params.MinTextVersion)
err := build.IsValidUnixVersion(*params.MinTextVersion)
if err != nil {
return err
}
internal/collector/set/set_test.go
+4 -4
View File
@@ -50,7 +50,7 @@ func TestSet(t *testing.T) {
LatestVersion: 4,
}
av := int32(2)
mv := int32(1)
mv := int64(1)
update := collectorset.SetParams{
ActiveVersion: &av,
MinCleanVersion: &mv,
@@ -91,7 +91,7 @@ func TestSet(t *testing.T) {
ClientID: uuid.New(),
}
av := int32(1)
mv := int32(1)
mv := int64(1)
update := collectorset.SetParams{
ActiveVersion: &av,
MinCleanVersion: &mv,
@@ -154,7 +154,7 @@ func TestSetByExternalId(t *testing.T) {
LatestVersion: 4,
}
av := int32(2)
mv := int32(1)
mv := int64(1)
update := collectorset.SetParams{
ActiveVersion: &av,
MinCleanVersion: &mv,
@@ -199,7 +199,7 @@ func TestSetByExternalId(t *testing.T) {
ClientID: uuid.New(),
}
av := int32(1)
mv := int32(1)
mv := int64(1)
update := collectorset.SetParams{
ActiveVersion: &av,
MinCleanVersion: &mv,
internal/collector/set/setprivate_test.go
+7 -15
View File
@@ -8,8 +8,6 @@ import (
"queryorchestration/internal/collector"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
cleanversion "queryorchestration/internal/document/clean/version"
textversion "queryorchestration/internal/document/text/version"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/queue/clientsync"
queuemock "queryorchestration/mocks/queue"
@@ -46,8 +44,8 @@ func TestGetSetParams(t *testing.T) {
},
}
minCleanV := int32(1)
minTextV := int32(1)
minCleanV := int64(1)
minTextV := int64(1)
aV := int32(3)
current := collector.Collector{
ActiveVersion: 1,
@@ -178,8 +176,8 @@ func TestSubmitSet(t *testing.T) {
svc: &Services{},
}
minCleanV := int32(2)
minTextV := int32(4)
minCleanV := int64(2)
minTextV := int64(4)
current := collector.Collector{
ClientID: uuid.New(),
ActiveVersion: 1,
@@ -342,13 +340,7 @@ func TestNormalizeActiveVersion(t *testing.T) {
}
func TestNormalizeCodeVersions(t *testing.T) {
cfg := &CollectorSetConfig{}
svc := Service{
svc: &Services{
CleanVersion: cleanversion.New(cfg),
TextVersion: textversion.New(cfg),
},
}
svc := Service{}
t.Run("all nil", func(t *testing.T) {
err := svc.normalizeCodeVersions(nil, nil)
@@ -375,7 +367,7 @@ func TestNormalizeCodeVersions(t *testing.T) {
current := collector.Collector{}
update := SetParams{}
cv := int32(1)
cv := int64(1)
update.MinCleanVersion = &cv
err := svc.normalizeCodeVersions(&current, &update)
assert.NoError(t, err)
@@ -387,7 +379,7 @@ func TestNormalizeCodeVersions(t *testing.T) {
current := collector.Collector{}
update := SetParams{}
tv := int32(1)
tv := int64(1)
update.MinTextVersion = &tv
err := svc.normalizeCodeVersions(&current, &update)
assert.NoError(t, err)
internal/database/repository/clean.sql.go
+2 -2
View File
@@ -45,7 +45,7 @@ INSERT INTO documentCleanEntries (cleanId, version) VALUES ($1, $2)
type AddDocumentCleanEntryParams struct {
Cleanid pgtype.UUID `db:"cleanid"`
Version int32 `db:"version"`
Version int64 `db:"version"`
}
// AddDocumentCleanEntry
@@ -133,7 +133,7 @@ type GetMostRecentDocumentCleanEntryRow struct {
Documentid pgtype.UUID `db:"documentid"`
Bucket *string `db:"bucket"`
Key *string `db:"key"`
Version int32 `db:"version"`
Version int64 `db:"version"`
Mimetype NullCleanmimetype `db:"mimetype"`
Fail NullCleanfailtype `db:"fail"`
}
internal/database/repository/clean_test.go
+2 -2
View File
@@ -88,7 +88,7 @@ func TestClean(t *testing.T) {
assert.Nil(t, clean.Bucket)
assert.Nil(t, clean.Key)
assert.Equal(t, fail, clean.Fail)
assert.Equal(t, int32(1), clean.Version)
assert.Equal(t, int64(1), clean.Version)
assert.Equal(t, failcleanid, clean.ID)
cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
@@ -115,7 +115,7 @@ func TestClean(t *testing.T) {
assert.Equal(t, bucket, *clean.Bucket)
assert.Equal(t, key, *clean.Key)
assert.Equal(t, mimetype, clean.Mimetype)
assert.Equal(t, int32(2), clean.Version)
assert.Equal(t, int64(2), clean.Version)
assert.Equal(t, cleanid, clean.ID)
docclean, err := queries.GetCleanEntryByDocId(ctx, id)
internal/database/repository/collector.sql.go
+2 -2
View File
@@ -176,7 +176,7 @@ INSERT INTO collectorMinCleanVersions (clientId, addedVersion, versionId) VALUES
type SetCollectorCleanVersionParams struct {
Clientid pgtype.UUID `db:"clientid"`
Addedversion int32 `db:"addedversion"`
Versionid int32 `db:"versionid"`
Versionid int64 `db:"versionid"`
}
// SetCollectorCleanVersion
@@ -194,7 +194,7 @@ INSERT INTO collectorMinTextVersions (clientId, addedVersion, versionId) VALUES
type SetCollectorTextVersionParams struct {
Clientid pgtype.UUID `db:"clientid"`
Addedversion int32 `db:"addedversion"`
Versionid int32 `db:"versionid"`
Versionid int64 `db:"versionid"`
}
// SetCollectorTextVersion
internal/database/repository/collector_test.go
+2 -2
View File
@@ -57,8 +57,8 @@ func TestCollector(t *testing.T) {
Externalid: "EXAMPLE",
})
assert.NoError(t, err)
minCleanVersion := int32(2)
minTextVersion := int32(4)
minCleanVersion := int64(2)
minTextVersion := int64(4)
coll, err := queries.GetCollectorByClientID(ctx, clientId)
assert.NoError(t, err)
internal/database/repository/models.go
+10 -10
View File
@@ -209,7 +209,7 @@ type Collectorlatestversion struct {
type Collectormincleanversion struct {
ID pgtype.UUID `db:"id"`
Clientid pgtype.UUID `db:"clientid"`
Versionid int32 `db:"versionid"`
Versionid int64 `db:"versionid"`
Addedversion int32 `db:"addedversion"`
Removedversion *int32 `db:"removedversion"`
}
@@ -217,7 +217,7 @@ type Collectormincleanversion struct {
type Collectormintextversion struct {
ID pgtype.UUID `db:"id"`
Clientid pgtype.UUID `db:"clientid"`
Versionid int32 `db:"versionid"`
Versionid int64 `db:"versionid"`
Addedversion int32 `db:"addedversion"`
Removedversion *int32 `db:"removedversion"`
}
@@ -250,7 +250,7 @@ type Currentcleanentry struct {
Documentid pgtype.UUID `db:"documentid"`
Bucket *string `db:"bucket"`
Key *string `db:"key"`
Version int32 `db:"version"`
Version int64 `db:"version"`
Mimetype NullCleanmimetype `db:"mimetype"`
Fail NullCleanfailtype `db:"fail"`
}
@@ -262,12 +262,12 @@ type Currentclientcansync struct {
type Currentcollectormincleanversion struct {
Clientid pgtype.UUID `db:"clientid"`
Mincleanversion int32 `db:"mincleanversion"`
Mincleanversion int64 `db:"mincleanversion"`
}
type Currentcollectormintextversion struct {
Clientid pgtype.UUID `db:"clientid"`
Mintextversion int32 `db:"mintextversion"`
Mintextversion int64 `db:"mintextversion"`
}
type Currentcollectorqueriesjsonagg struct {
@@ -286,7 +286,7 @@ type Currenttextentry struct {
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
Version int64 `db:"version"`
Cleanentryid pgtype.UUID `db:"cleanentryid"`
}
@@ -308,7 +308,7 @@ type Documentclean struct {
type Documentcleanentry struct {
ID pgtype.UUID `db:"id"`
Cleanid pgtype.UUID `db:"cleanid"`
Version int32 `db:"version"`
Version int64 `db:"version"`
}
type Documententry struct {
@@ -321,15 +321,15 @@ type Documententry struct {
type Documenttextextraction struct {
ID pgtype.UUID `db:"id"`
Cleanentryid pgtype.UUID `db:"cleanentryid"`
Version int32 `db:"version"`
Version int64 `db:"version"`
Bucket string `db:"bucket"`
Key string `db:"key"`
}
type Fullactivecollector struct {
Clientid pgtype.UUID `db:"clientid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
Mincleanversion int64 `db:"mincleanversion"`
Mintextversion int64 `db:"mintextversion"`
Activeversion int32 `db:"activeversion"`
Latestversion int32 `db:"latestversion"`
Fields []byte `db:"fields"`
internal/database/repository/text.sql.go
+1 -1
View File
@@ -16,7 +16,7 @@ INSERT INTO documentTextExtractions (version, bucket, key, cleanEntryId) VALUES
`
type AddDocumentTextEntryParams struct {
Version int32 `db:"version"`
Version int64 `db:"version"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Cleanentryid pgtype.UUID `db:"cleanentryid"`
internal/database/repository/text_test.go
+1 -1
View File
@@ -84,7 +84,7 @@ func TestTextExtraction(t *testing.T) {
assert.Equal(t, id, text.Documentid)
assert.Equal(t, bucket, text.Bucket)
assert.Equal(t, key, text.Key)
assert.Equal(t, int32(1), text.Version)
assert.Equal(t, int64(1), text.Version)
assert.NotEqual(t, pgtype.UUID{}, text.ID)
assert.True(t, text.ID.Valid)
}
internal/document/clean/clean.go
+2 -1
View File
@@ -10,6 +10,7 @@ import (
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
"queryorchestration/internal/serviceconfig/build"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid"
@@ -121,7 +122,7 @@ func (s *Service) clean(ctx context.Context, id uuid.UUID) error {
func (s *Service) storeClean(ctx context.Context, id uuid.UUID, out *ExecuteCleanResponse) error {
docId := database.MustToDBUUID(id)
version := s.svc.Version.GetVersion()
version := build.GetVersionUnixTimestamp()
params := &repository.AddDocumentCleanParams{
Documentid: docId,
internal/document/clean/clean_test.go
+6 -13
View File
@@ -9,7 +9,6 @@ import (
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
cleanversion "queryorchestration/internal/document/clean/version"
objectstoremock "queryorchestration/mocks/objectstore"
"github.com/stretchr/testify/require"
@@ -34,9 +33,6 @@ func TestClean(t *testing.T) {
svc := Service{
cfg: cfg,
svc: &Services{
Version: cleanversion.New(cfg),
},
}
doc := document.Document{
@@ -75,7 +71,7 @@ func TestClean(t *testing.T) {
pgxmock.NewRows([]string{"id"}).
AddRow(cleanid),
)
pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(cleanid, int32(1)).
pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(cleanid, pgxmock.AnyArg()).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
@@ -175,9 +171,6 @@ func TestStoreClean(t *testing.T) {
svc := Service{
cfg: cfg,
svc: &Services{
Version: cleanversion.New(cfg),
},
}
doc := document.Document{
@@ -210,7 +203,7 @@ func TestStoreClean(t *testing.T) {
pgxmock.NewRows([]string{"id"}).
AddRow(cleanid),
)
pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(cleanid, int32(1)).
pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(cleanid, pgxmock.AnyArg()).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
@@ -233,7 +226,7 @@ func TestStoreClean(t *testing.T) {
pool.ExpectQuery("name: GetMostRecentDocumentCleanEntry :one").WithArgs(dbid).
WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(database.MustToDBUUID(uuid.New()), dbid, nil, nil, int32(1), nil, repository.CleanfailtypeInvalidRead),
AddRow(database.MustToDBUUID(uuid.New()), dbid, nil, nil, int64(1), nil, repository.CleanfailtypeInvalidRead),
)
cleanid := database.MustToDBUUID(uuid.New())
pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &inloc.Bucket, &inloc.Key, dbmimetype, repository.NullCleanfailtype{}).
@@ -241,7 +234,7 @@ func TestStoreClean(t *testing.T) {
pgxmock.NewRows([]string{"id"}).
AddRow(cleanid),
)
pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(cleanid, int32(1)).
pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(cleanid, pgxmock.AnyArg()).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
@@ -260,12 +253,12 @@ func TestStoreClean(t *testing.T) {
pool.ExpectQuery("name: GetMostRecentDocumentCleanEntry :one").WithArgs(dbid).
WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(cleanid, dbid, nil, nil, int32(1), nil, repository.NullCleanfailtype{
AddRow(cleanid, dbid, nil, nil, int64(1), nil, repository.NullCleanfailtype{
Valid: true,
Cleanfailtype: repository.CleanfailtypeInvalidMimetype,
}),
)
pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(cleanid, int32(1)).
pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(cleanid, pgxmock.AnyArg()).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
internal/document/clean/content_test.go
-4
View File
@@ -7,7 +7,6 @@ import (
"testing"
"queryorchestration/internal/document"
cleanversion "queryorchestration/internal/document/clean/version"
objectstoremock "queryorchestration/mocks/objectstore"
"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -25,9 +24,6 @@ func TestGetContent(t *testing.T) {
svc := Service{
cfg: cfg,
svc: &Services{
Version: cleanversion.New(cfg),
},
}
inloc := document.Location{
internal/document/clean/create_test.go
+1 -5
View File
@@ -10,7 +10,6 @@ import (
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
cleanversion "queryorchestration/internal/document/clean/version"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documenttext"
@@ -49,9 +48,6 @@ func TestCreate(t *testing.T) {
svc := Service{
cfg: cfg,
svc: &Services{
Version: cleanversion.New(cfg),
},
}
doc := document.Document{
@@ -94,7 +90,7 @@ func TestCreate(t *testing.T) {
pgxmock.NewRows([]string{"id"}).
AddRow(cleanid),
)
pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(cleanid, int32(1)).
pool.ExpectExec("name: AddDocumentCleanEntry :exec").WithArgs(cleanid, pgxmock.AnyArg()).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
internal/document/clean/service.go
+1 -8
View File
@@ -1,7 +1,6 @@
package documentclean
import (
cleanversion "queryorchestration/internal/document/clean/version"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documenttext"
@@ -13,18 +12,12 @@ type ConfigProvider interface {
objectstore.ConfigProvider
}
type Services struct {
Version *cleanversion.Service
}
type Service struct {
cfg ConfigProvider
svc *Services
}
func New(cfg ConfigProvider, svc *Services) *Service {
func New(cfg ConfigProvider) *Service {
return &Service{
cfg: cfg,
svc: svc,
}
}
internal/document/clean/service_test.go
+1 -1
View File
@@ -19,6 +19,6 @@ type DocCleanConfig struct {
func TestService(t *testing.T) {
cfg := &DocCleanConfig{}
svc := documentclean.New(cfg, nil)
svc := documentclean.New(cfg)
assert.NotNil(t, svc)
}
internal/document/clean/version/service.go
-15
View File
@@ -1,15 +0,0 @@
package cleanversion
import (
"queryorchestration/internal/serviceconfig"
)
type Service struct {
cfg serviceconfig.ConfigProvider
}
func New(cfg serviceconfig.ConfigProvider) *Service {
return &Service{
cfg,
}
}
internal/document/clean/version/service_test.go
-14
View File
@@ -1,14 +0,0 @@
package cleanversion_test
import (
"testing"
cleanversion "queryorchestration/internal/document/clean/version"
"github.com/stretchr/testify/assert"
)
func TestNew(t *testing.T) {
svc := cleanversion.New(nil)
assert.NotNil(t, svc)
}
internal/document/clean/version/version.go
-19
View File
@@ -1,19 +0,0 @@
package cleanversion
import (
"fmt"
)
func (s *Service) GetVersion() int32 {
return 1
}
func (s *Service) IsValidVersion(v int32) error {
currentVersion := s.GetVersion()
if v < 1 || v > currentVersion {
return fmt.Errorf("document clean code version must be in the range 0 < version <= %d", currentVersion)
}
return nil
}
internal/document/clean/version/version_test.go
-27
View File
@@ -1,27 +0,0 @@
package cleanversion_test
import (
"testing"
cleanversion "queryorchestration/internal/document/clean/version"
"queryorchestration/internal/serviceconfig"
"github.com/stretchr/testify/assert"
)
func TestIsValidVersion(t *testing.T) {
cfg := &serviceconfig.BaseConfig{}
svc := cleanversion.New(cfg)
assert.Nil(t, svc.IsValidVersion(1))
assert.Error(t, svc.IsValidVersion(2))
assert.Error(t, svc.IsValidVersion(0))
assert.Error(t, svc.IsValidVersion(-1))
}
func TestGetVersion(t *testing.T) {
cfg := &serviceconfig.BaseConfig{}
svc := cleanversion.New(cfg)
assert.Equal(t, int32(1), svc.GetVersion())
}
internal/document/text/create_test.go
+2 -6
View File
@@ -8,7 +8,6 @@ import (
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
textversion "queryorchestration/internal/document/text/version"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/queue/querysync"
queuemock "queryorchestration/mocks/queue"
@@ -42,9 +41,6 @@ func TestCreate(t *testing.T) {
svc := Service{
cfg: cfg,
svc: &Services{
Version: textversion.New(cfg),
},
}
t.Run("valid", func(t *testing.T) {
@@ -72,7 +68,7 @@ func TestCreate(t *testing.T) {
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(cleanId, database.MustToDBUUID(id), &inloc.Bucket, &inloc.Key, int32(1), dbmimetype, repository.NullCleanfailtype{}),
)
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(int32(1), inloc.Bucket, inloc.Key, cleanId).
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(pgxmock.AnyArg(), inloc.Bucket, inloc.Key, cleanId).
WillReturnResult(pgxmock.NewResult("", 1))
mockSQS.EXPECT().
@@ -100,7 +96,7 @@ func TestCreate(t *testing.T) {
)
pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(database.MustToDBUUID(id), database.MustToDBUUID(uuid.New()), nil, nil, int32(1), nil, fail),
AddRow(database.MustToDBUUID(id), database.MustToDBUUID(uuid.New()), nil, nil, int64(1), nil, fail),
)
err = svc.Extract(ctx, id)
internal/document/text/extract.go
+2 -1
View File
@@ -7,6 +7,7 @@ import (
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
"queryorchestration/internal/serviceconfig/build"
"github.com/google/uuid"
)
@@ -31,7 +32,7 @@ func (s *Service) extract(ctx context.Context, cleanId uuid.UUID) error {
return err
}
version := s.svc.Version.GetVersion()
version := build.GetVersionUnixTimestamp()
err = s.cfg.GetDBQueries().AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: version,
internal/document/text/extract_test.go
+3 -10
View File
@@ -7,7 +7,6 @@ import (
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
textversion "queryorchestration/internal/document/text/version"
"github.com/stretchr/testify/require"
@@ -27,9 +26,6 @@ func TestExtract(t *testing.T) {
svc := Service{
cfg: cfg,
svc: &Services{
Version: textversion.New(cfg),
},
}
t.Run("successful clean", func(t *testing.T) {
@@ -48,9 +44,9 @@ func TestExtract(t *testing.T) {
}
pool.ExpectQuery("name: GetCleanEntry :one").WithArgs(dbCleanId).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(dbCleanId, database.MustToDBUUID(id), &inloc.Bucket, &inloc.Key, int32(1), dbmimetype, repository.NullCleanfailtype{}),
AddRow(dbCleanId, database.MustToDBUUID(id), &inloc.Bucket, &inloc.Key, int64(1), dbmimetype, repository.NullCleanfailtype{}),
)
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(int32(1), inloc.Bucket, inloc.Key, dbCleanId).
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(pgxmock.AnyArg(), inloc.Bucket, inloc.Key, dbCleanId).
WillReturnResult(pgxmock.NewResult("", 1))
err = svc.extract(ctx, cleanId)
@@ -66,7 +62,7 @@ func TestExtract(t *testing.T) {
}
pool.ExpectQuery("name: GetCleanEntry :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(cleanId, database.MustToDBUUID(id), nil, nil, int32(1), nil, fail),
AddRow(cleanId, database.MustToDBUUID(id), nil, nil, int64(1), nil, fail),
)
err = svc.extract(ctx, id)
@@ -84,9 +80,6 @@ func TestExecuteExtraction(t *testing.T) {
svc := Service{
cfg: cfg,
svc: &Services{
Version: textversion.New(cfg),
},
}
ctx := context.Background()
internal/document/text/service.go
+1 -8
View File
@@ -1,7 +1,6 @@
package documenttext
import (
textversion "queryorchestration/internal/document/text/version"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/queue/querysync"
)
@@ -11,18 +10,12 @@ type ConfigProvider interface {
querysync.ConfigProvider
}
type Services struct {
Version *textversion.Service
}
type Service struct {
cfg ConfigProvider
svc *Services
}
func New(cfg ConfigProvider, svc *Services) *Service {
func New(cfg ConfigProvider) *Service {
return &Service{
cfg: cfg,
svc: svc,
}
}
internal/document/text/service_test.go
+1 -4
View File
@@ -4,7 +4,6 @@ import (
"testing"
documenttext "queryorchestration/internal/document/text"
textversion "queryorchestration/internal/document/text/version"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/queue/querysync"
@@ -18,8 +17,6 @@ type DocTextConfig struct {
func TestService(t *testing.T) {
cfg := &DocTextConfig{}
svc := documenttext.New(cfg, &documenttext.Services{
Version: textversion.New(cfg),
})
svc := documenttext.New(cfg)
assert.NotNil(t, svc)
}
internal/document/text/version/service.go
-15
View File
@@ -1,15 +0,0 @@
package textversion
import (
"queryorchestration/internal/serviceconfig"
)
type Service struct {
cfg serviceconfig.ConfigProvider
}
func New(cfg serviceconfig.ConfigProvider) *Service {
return &Service{
cfg,
}
}
internal/document/text/version/service_test.go
-14
View File
@@ -1,14 +0,0 @@
package textversion_test
import (
"testing"
textversion "queryorchestration/internal/document/clean/version"
"github.com/stretchr/testify/assert"
)
func TestNew(t *testing.T) {
svc := textversion.New(nil)
assert.NotNil(t, svc)
}
internal/document/text/version/version.go
-19
View File
@@ -1,19 +0,0 @@
package textversion
import (
"fmt"
)
func (s *Service) GetVersion() int32 {
return 1
}
func (s *Service) IsValidVersion(v int32) error {
currentVersion := s.GetVersion()
if v < 1 || v > currentVersion {
return fmt.Errorf("document text code version must be in the range 0 < version <= %d", currentVersion)
}
return nil
}
internal/document/text/version/version_test.go
-27
View File
@@ -1,27 +0,0 @@
package textversion_test
import (
"testing"
textversion "queryorchestration/internal/document/clean/version"
"queryorchestration/internal/serviceconfig"
"github.com/stretchr/testify/assert"
)
func TestIsValidVersion(t *testing.T) {
cfg := &serviceconfig.BaseConfig{}
svc := textversion.New(cfg)
assert.Nil(t, svc.IsValidVersion(1))
assert.Error(t, svc.IsValidVersion(2))
assert.Error(t, svc.IsValidVersion(0))
assert.Error(t, svc.IsValidVersion(-1))
}
func TestGetVersion(t *testing.T) {
cfg := &serviceconfig.BaseConfig{}
svc := textversion.New(cfg)
assert.Equal(t, int32(1), svc.GetVersion())
}
internal/server/server.go
+2 -2
View File
@@ -3,7 +3,7 @@ package server
import (
"context"
"queryorchestration/internal/serviceconfig/observability"
"queryorchestration/internal/serviceconfig/build"
"queryorchestration/internal/database/migrations"
"queryorchestration/internal/serviceconfig"
@@ -43,7 +43,7 @@ func New(ctx context.Context, cfg Config) (func() error, error) {
cfg.PrintConfig(DEFAULT_SECRET_PREFIX)
closeTracer := cfg.SetOtel(ctx)
version := observability.GetVersion()
version := build.GetVersion()
cfg.GetLogger().Info("Starting", "version", version)
err := migrations.Run(ctx, cfg)
internal/serviceconfig/build/build.go
+76
View File
@@ -0,0 +1,76 @@
package build
import (
"fmt"
"runtime/debug"
"sync"
"time"
)
const (
SystemName = "doczy"
)
func IsValidUnixVersion(v int64) error {
currentVersion := GetVersionUnixTimestamp()
if v < 1 || v > currentVersion {
return fmt.Errorf("version must be in the range 0 < version <= %d", currentVersion)
}
return nil
}
var startupTime time.Time
var startupTimeOnce sync.Once
func GetVersionTimestamp() time.Time {
startupTimeOnce.Do(func() {
info, ok := debug.ReadBuildInfo()
if !ok {
startupTime = time.Now()
return
}
// Look for vcs.time in build settings
for _, setting := range info.Settings {
if setting.Key == "vcs.time" {
// Parse the build timestamp
buildTime, err := time.Parse(time.RFC3339, setting.Value)
if err == nil {
startupTime = buildTime
return
}
break
}
}
// Return current time if no build time found
startupTime = time.Now()
})
return startupTime
}
func GetVersionUnixTimestamp() int64 {
return GetVersionTimestamp().UnixMilli()
}
// GetVersion returns the version of the application in the Golang build info format.
// This is a raw format and will change depending on the build environment and the git tags.
// When building from a commit with an associated Git tag (e.g., v1.2.3),
//
// the runtime version uses the tag name without a timestamp.
//
// The "+dirty" suffix is appended when there are uncommitted changes
//
// present in the version control system (VCS) at the time of building.
//
// An example of a version v0.0.2-0.20250310175756-a91372dc0602+dirty.
func GetVersion() string {
info, ok := debug.ReadBuildInfo()
if !ok {
return "unknown"
}
return info.Main.Version
}
internal/serviceconfig/build/build_test.go
+74
View File
@@ -0,0 +1,74 @@
package build_test
import (
"testing"
"time"
"queryorchestration/internal/serviceconfig/build"
"github.com/stretchr/testify/assert"
)
func TestGetVersionUnixTimestamp(t *testing.T) {
t1 := build.GetVersionUnixTimestamp()
assert.NotZero(t, t1)
time.Sleep(time.Millisecond * 100)
t2 := build.GetVersionUnixTimestamp()
assert.Equal(t, t1, t2)
}
func TestIsValidUnixVersion(t *testing.T) {
assert.NotNil(t, build.IsValidUnixVersion(0))
assert.NotNil(t, build.IsValidUnixVersion(time.Now().Add(time.Second).UnixMilli()))
assert.Nil(t, build.IsValidUnixVersion(1))
}
func TestGetVersionTimestamp(t *testing.T) {
// Test 1: First call should set the timestamp
t1 := build.GetVersionTimestamp()
if t1.IsZero() {
t.Error("GetVersionTimestamp returned zero time")
}
// Test 2: Second call should return exactly the same time
time.Sleep(time.Millisecond * 100) // Small delay to ensure time would be different if not cached
t2 := build.GetVersionTimestamp()
if !t1.Equal(t2) {
t.Errorf("Timestamps should be identical but got t1=%v, t2=%v", t1, t2)
}
// Test 3: Even after a longer delay, should still return the same time
time.Sleep(time.Second) // Longer delay
t3 := build.GetVersionTimestamp()
if !t1.Equal(t3) {
t.Errorf("Timestamp changed after delay: original=%v, new=%v", t1, t3)
}
// Test 4: Returned time should not be in the future
if t1.After(time.Now()) {
t.Errorf("Timestamp is in the future: %v", t1)
}
}
func TestGetVersion(t *testing.T) {
// Test 1: Basic call
version := build.GetVersion()
if version == "" {
t.Error("Version should not be empty")
}
// Test 2: Should either be "unknown" or follow semver-like pattern
if version != "unknown" {
// This is a loose check - you might want to make it more strict
if len(version) < 3 {
t.Errorf("Version '%s' seems too short to be valid", version)
}
}
// Test 3: Multiple calls should be consistent
version2 := build.GetVersion()
if version != version2 {
t.Errorf("Version not consistent between calls: '%s' != '%s'", version, version2)
}
}
internal/serviceconfig/observability/config.go
+2 -59
View File
@@ -3,11 +3,8 @@ package observability
import (
"context"
"log/slog"
"sync"
"time"
"runtime/debug"
"queryorchestration/internal/serviceconfig/build"
"queryorchestration/internal/serviceconfig/observability/prometheus"
"go.opentelemetry.io/otel"
@@ -15,60 +12,6 @@ import (
sdktrace "go.opentelemetry.io/otel/sdk/trace"
)
const (
SystemName = "doczy"
)
var startupTime time.Time
var startupTimeOnce sync.Once
func GetVersionTimestamp() time.Time {
startupTimeOnce.Do(func() {
info, ok := debug.ReadBuildInfo()
if !ok {
startupTime = time.Now()
return
}
// Look for vcs.time in build settings
for _, setting := range info.Settings {
if setting.Key == "vcs.time" {
// Parse the build timestamp
buildTime, err := time.Parse(time.RFC3339, setting.Value)
if err == nil {
startupTime = buildTime
return
}
break
}
}
// Return current time if no build time found
startupTime = time.Now()
})
return startupTime
}
// GetVersion returns the version of the application in the Golang build info format.
// This is a raw format and will change depending on the build environment and the git tags.
// When building from a commit with an associated Git tag (e.g., v1.2.3),
//
// the runtime version uses the tag name without a timestamp.
//
// The "+dirty" suffix is appended when there are uncommitted changes
//
// present in the version control system (VCS) at the time of building.
//
// An example of a version v0.0.2-0.20250310175756-a91372dc0602+dirty.
func GetVersion() string {
info, ok := debug.ReadBuildInfo()
if !ok {
return "unknown"
}
return info.Main.Version
}
type ConfigProvider interface {
IsOtelEnabled() bool
SetOtel(context.Context) func() error
@@ -127,7 +70,7 @@ func InitializeMetrics(ctx context.Context, cfg ConfigProvider) error {
standardDefinitions := prometheus.NewStandardMetricDefinitions()
config := prometheus.MetricsConfig{
Port: 8080,
Namespace: SystemName,
Namespace: build.SystemName,
Subsystem: SubSystem,
ServeHttp: true,
Definitions: standardDefinitions,
internal/serviceconfig/observability/config_test.go
-50
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"testing"
"time"
"queryorchestration/internal/serviceconfig/observability"
@@ -33,52 +32,3 @@ func TestSetOtel(t *testing.T) {
}
}()
}
func TestGetVersionTimestamp(t *testing.T) {
// Test 1: First call should set the timestamp
t1 := observability.GetVersionTimestamp()
if t1.IsZero() {
t.Error("GetVersionTimestamp returned zero time")
}
// Test 2: Second call should return exactly the same time
time.Sleep(time.Millisecond * 100) // Small delay to ensure time would be different if not cached
t2 := observability.GetVersionTimestamp()
if !t1.Equal(t2) {
t.Errorf("Timestamps should be identical but got t1=%v, t2=%v", t1, t2)
}
// Test 3: Even after a longer delay, should still return the same time
time.Sleep(time.Second) // Longer delay
t3 := observability.GetVersionTimestamp()
if !t1.Equal(t3) {
t.Errorf("Timestamp changed after delay: original=%v, new=%v", t1, t3)
}
// Test 4: Returned time should not be in the future
if t1.After(time.Now()) {
t.Errorf("Timestamp is in the future: %v", t1)
}
}
func TestGetVersion(t *testing.T) {
// Test 1: Basic call
version := observability.GetVersion()
if version == "" {
t.Error("Version should not be empty")
}
// Test 2: Should either be "unknown" or follow semver-like pattern
if version != "unknown" {
// This is a loose check - you might want to make it more strict
if len(version) < 3 {
t.Errorf("Version '%s' seems too short to be valid", version)
}
}
// Test 3: Multiple calls should be consistent
version2 := observability.GetVersion()
if version != version2 {
t.Errorf("Version not consistent between calls: '%s' != '%s'", version, version2)
}
}
pkg/queryService/api.gen.go
+1 -1
View File
@@ -95,7 +95,7 @@ type ClientUpdate struct {
}
// CodeVersion The desired code version.
type CodeVersion = int32
type CodeVersion = int64
// Collector Collector model.
type Collector struct {
serviceAPIs/queryService.yaml
+1 -1
View File
@@ -570,7 +570,7 @@ components:
CodeVersion:
type: integer
format: int32
format: int64
description: The desired code version.
Version:
test/queryService/collectorservice_test.go
+13 -16
View File
@@ -2,9 +2,6 @@ package endtoend
import (
"context"
"log"
"os"
"path"
"regexp"
"testing"
@@ -26,8 +23,7 @@ func TestCollectorService(t *testing.T) {
ctx := context.Background()
cfg := &CollectorConfig{}
test.SetCfgProvider(t, cfg)
cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../.."))
test.SetCfgProviderWithBasePath(t, cfg, "../..")
network, ncleanup := test.CreateNetwork(t, ctx)
defer ncleanup()
@@ -38,8 +34,7 @@ func TestCollectorService(t *testing.T) {
})
defer clean()
err := cfg.SetQueueClient(ctx)
assert.NoError(t, err)
test.SetQueueClient(t, ctx, cfg)
clientsyncurl := test.CreateQueue(t, ctx, cfg, test.ClientSyncRunner)
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
@@ -83,8 +78,8 @@ func TestCollectorService(t *testing.T) {
assert.Equal(t, id, collRes.JSON200.ClientId)
assert.Equal(t, int32(0), collRes.JSON200.ActiveVersion)
assert.Equal(t, int32(0), collRes.JSON200.LatestVersion)
assert.Equal(t, int32(0), collRes.JSON200.MinimumCleanerVersion)
assert.Equal(t, int32(0), collRes.JSON200.MinimumTextVersion)
assert.Equal(t, int64(0), collRes.JSON200.MinimumCleanerVersion)
assert.Equal(t, int64(0), collRes.JSON200.MinimumTextVersion)
assert.Len(t, collRes.JSON200.Fields, 0)
av := int32(1)
@@ -94,23 +89,25 @@ func TestCollectorService(t *testing.T) {
QueryId: jsonRes.JSON201.Id,
},
}
minVersion := int64(1000)
uRes, err := client.SetCollectorByClientIdWithResponse(ctx, id, queryservice.CollectorSet{
ActiveVersion: &av,
Fields: &fields,
ActiveVersion: &av,
MinimumCleanerVersion: &minVersion,
MinimumTextVersion: &minVersion,
Fields: &fields,
})
assert.NoError(t, err)
log.Print(string(uRes.Body))
assert.Equal(t, 200, uRes.StatusCode())
test.AssertMessageBody(t, ctx, cfg, clientsyncurl, regexp.MustCompile(`{"id":".+"}`))
collRes, err = client.GetCollectorByClientIdWithResponse(ctx, id)
assert.NoError(t, err)
assert.Equal(t, id, collRes.JSON200.ClientId)
assert.Equal(t, int32(1), collRes.JSON200.ActiveVersion)
assert.Equal(t, int32(1), collRes.JSON200.LatestVersion)
assert.Equal(t, int32(0), collRes.JSON200.MinimumCleanerVersion)
assert.Equal(t, int32(0), collRes.JSON200.MinimumTextVersion)
assert.Equal(t, minVersion, collRes.JSON200.MinimumCleanerVersion)
assert.Equal(t, minVersion, collRes.JSON200.MinimumTextVersion)
assert.Len(t, collRes.JSON200.Fields, 1)
assert.ElementsMatch(t, fields, collRes.JSON200.Fields)
test.AssertMessageBody(t, ctx, cfg, clientsyncurl, regexp.MustCompile(`{"id":".+"}`))
}