jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/jobcollector (pull request #30)

Browse Source 
Initial Job Collector (changes pending)

* movearroundtocleancollector

* internalgetfunctions

* completecollectorquery

* simplify

* fixtests

* addvendor

* noplaceholder
This commit is contained in:
Michael McGuinness
2025-01-21 12:28:46 +00:00
parent b888e3450f
commit 4ccb980593
46 changed files with 951 additions and 655 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -14,7 +14,7 @@ Using the following project as a baseline: https://github.com/golang-standards/p
sudo groupadd docker
sudo usermod -aG docker $USER
```
- Run `touch .env` to create `.env` file from custom environment variables
- Run `touch .env` to create `.env` file for custom environment variables
- Run `devbox shell` to enter the environment
- Run `task fullsuite` to run all tests that ensure the current state
api/queryRunner/queryrunner_test.go
+9 -9
View File
@@ -6,6 +6,7 @@ import (
controllers "queryorchestration/api/queryRunner"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/job/collector"
"queryorchestration/internal/query/document"
"testing"
@@ -18,9 +19,6 @@ import (
)
func TestQueryRunner(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
pool, err := pgxmock.NewPool()
@@ -33,7 +31,9 @@ func TestQueryRunner(t *testing.T) {
Pool: pool,
}
svc := document.New(db)
svc := document.New(db, &document.Services{
Collector: collector.New(db),
})
runner := controllers.NewQueryRunner(svc, validator.New())
assert.NotNil(t, runner)
@@ -58,20 +58,20 @@ func TestQueryRunner(t *testing.T) {
qV := int32(1)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)).
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(collectorID, database.MustToDBUUID(doc.JobID), minCleanVersion, minTextVersion),
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(collectorID, database.MustToDBUUID(doc.JobID), minCleanVersion, minTextVersion, int32(1), int32(2), []byte("")),
)
pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}).
AddRow(collectorID, queryID, qV),
)
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(collectorID).
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(collectorID).
WillReturnRows(
pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
AddRow(collectorID, queryID, repository.NullQuerytype{Querytype: repository.QuerytypeContextFull, Valid: true}, &qV, []pgtype.UUID{}),
AddRow(collectorID, queryID, repository.QuerytypeContextFull, qV, []pgtype.UUID{}),
)
err = runner.Process(ctx, msg)
api/queryService/api.gen.go
+89 -76
View File
@@ -99,20 +99,47 @@ type IdMessage struct {
// JobCollector JobCollector model.
type JobCollector struct {
// ExampleProperty Example property for JobCollector.
ExampleProperty *string `json:"example_property,omitempty"`
// ActiveVersion The active version of the collector.
ActiveVersion int32 `json:"active_version"`
// Fields The fields in the job collector.
Fields []JobCollectorField `json:"fields"`
// JobId The ID of the associated job.
JobId string `json:"job_id"`
// LatestVersion The latest version of the collector.
LatestVersion int32 `json:"latest_version"`
// MinimumCleanerVersion The minimum version for the document cleaner.
MinimumCleanerVersion int32 `json:"minimum_cleaner_version"`
// MinimumTextVersion The minimum version for the text parser.
MinimumTextVersion int32 `json:"minimum_text_version"`
}
// JobCollectorCreate Payload for creating a JobCollector.
type JobCollectorCreate struct {
// ExampleProperty Example property for JobCollectorCreate.
ExampleProperty *string `json:"example_property,omitempty"`
// JobCollectorField The field properties for the job collector.
type JobCollectorField struct {
// Name The output field name.
Name string `json:"name"`
// QueryId The query id that will populate the result.
QueryId string `json:"query_id"`
}
// JobCollectorUpdate Payload for updating a JobCollector.
type JobCollectorUpdate struct {
// ExampleProperty Example property for JobCollectorUpdate.
ExampleProperty *string `json:"example_property,omitempty"`
// ActiveVersion The active version of the collector.
ActiveVersion *int32 `json:"active_version,omitempty"`
// Fields The fields in the job collector.
Fields *[]JobCollectorField `json:"fields,omitempty"`
// MinimumCleanerVersion The minimum version for the document cleaner.
MinimumCleanerVersion *int32 `json:"minimum_cleaner_version,omitempty"`
// MinimumTextVersion The minimum version for the text parser.
MinimumTextVersion *int32 `json:"minimum_text_version,omitempty"`
}
// ListQueries defines model for ListQueries.
@@ -187,11 +214,8 @@ type QueryUpdate struct {
// TriggerExportJSONRequestBody defines body for TriggerExport for application/json ContentType.
type TriggerExportJSONRequestBody = ExportTrigger
// CreateJobCollectorJSONRequestBody defines body for CreateJobCollector for application/json ContentType.
type CreateJobCollectorJSONRequestBody = JobCollectorCreate
// UpdateJobCollectorJSONRequestBody defines body for UpdateJobCollector for application/json ContentType.
type UpdateJobCollectorJSONRequestBody = JobCollectorUpdate
// UpdateJobCollectorByJobIdJSONRequestBody defines body for UpdateJobCollectorByJobId for application/json ContentType.
type UpdateJobCollectorByJobIdJSONRequestBody = JobCollectorUpdate
// CreateQueryJSONRequestBody defines body for CreateQuery for application/json ContentType.
type CreateQueryJSONRequestBody = QueryCreate
@@ -205,20 +229,17 @@ type TestQueryJSONRequestBody = QueryTestRequest
// ServerInterface represents all server handlers.
type ServerInterface interface {
// Trigger an export
// (POST /export)
// (POST /job/export)
TriggerExport(ctx echo.Context) error
// Check export state.
// (GET /export/{id})
// (GET /job/export/{id})
ExportState(ctx echo.Context, id string) error
// Create a job collector
// (POST /job-collectors)
CreateJobCollector(ctx echo.Context) error
// Get a job collector by ID
// (GET /job-collectors/{id})
GetJobCollectorById(ctx echo.Context, id string) error
// (GET /job/{id}/collector)
GetJobCollectorByJobId(ctx echo.Context, id string) error
// Update a job collector
// (PUT /job-collectors/{id})
UpdateJobCollector(ctx echo.Context, id string) error
// (PATCH /job/{id}/collector)
UpdateJobCollectorByJobId(ctx echo.Context, id string) error
// List queries
// (GET /queries)
ListQueries(ctx echo.Context) error
@@ -269,17 +290,8 @@ func (w *ServerInterfaceWrapper) ExportState(ctx echo.Context) error {
return err
}
// CreateJobCollector converts echo context to params.
func (w *ServerInterfaceWrapper) CreateJobCollector(ctx echo.Context) error {
var err error
// Invoke the callback with all the unmarshaled arguments
err = w.Handler.CreateJobCollector(ctx)
return err
}
// GetJobCollectorById converts echo context to params.
func (w *ServerInterfaceWrapper) GetJobCollectorById(ctx echo.Context) error {
// GetJobCollectorByJobId converts echo context to params.
func (w *ServerInterfaceWrapper) GetJobCollectorByJobId(ctx echo.Context) error {
var err error
// ------------- Path parameter "id" -------------
var id string
@@ -290,12 +302,12 @@ func (w *ServerInterfaceWrapper) GetJobCollectorById(ctx echo.Context) error {
}
// Invoke the callback with all the unmarshaled arguments
err = w.Handler.GetJobCollectorById(ctx, id)
err = w.Handler.GetJobCollectorByJobId(ctx, id)
return err
}
// UpdateJobCollector converts echo context to params.
func (w *ServerInterfaceWrapper) UpdateJobCollector(ctx echo.Context) error {
// UpdateJobCollectorByJobId converts echo context to params.
func (w *ServerInterfaceWrapper) UpdateJobCollectorByJobId(ctx echo.Context) error {
var err error
// ------------- Path parameter "id" -------------
var id string
@@ -306,7 +318,7 @@ func (w *ServerInterfaceWrapper) UpdateJobCollector(ctx echo.Context) error {
}
// Invoke the callback with all the unmarshaled arguments
err = w.Handler.UpdateJobCollector(ctx, id)
err = w.Handler.UpdateJobCollectorByJobId(ctx, id)
return err
}
@@ -420,11 +432,10 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL
Handler: si,
}
router.POST(baseURL+"/export", wrapper.TriggerExport)
router.GET(baseURL+"/export/:id", wrapper.ExportState)
router.POST(baseURL+"/job-collectors", wrapper.CreateJobCollector)
router.GET(baseURL+"/job-collectors/:id", wrapper.GetJobCollectorById)
router.PUT(baseURL+"/job-collectors/:id", wrapper.UpdateJobCollector)
router.POST(baseURL+"/job/export", wrapper.TriggerExport)
router.GET(baseURL+"/job/export/:id", wrapper.ExportState)
router.GET(baseURL+"/job/:id/collector", wrapper.GetJobCollectorByJobId)
router.PATCH(baseURL+"/job/:id/collector", wrapper.UpdateJobCollectorByJobId)
router.GET(baseURL+"/queries", wrapper.ListQueries)
router.POST(baseURL+"/queries", wrapper.CreateQuery)
router.DELETE(baseURL+"/queries/:id", wrapper.DeprecateQuery)
@@ -437,40 +448,42 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL
// Base64 encoded, gzipped, json marshaled Swagger object
var swaggerSpec = []string{
"H4sIAAAAAAAC/9RaX2/bOBL/KgTvHn12ut0nv3WTtHDR2/RS97DAojBocmQzS5MKSbn1Bv7uC5KSTEmU",
"LTdJkT61lsgZzvx+84ejPGCqNrmSIK3B0wds6Bo2xP/3+luutL0CS7jwDxgYqnluuZJ4ij+SnVCEoUxp",
"BH4pspqvVqCRBpMraWCMRzjXKgdtOXgRd2q54KwrbL4GdKeWiDOkQRDLt4CsQnYNpWwnyu5ywFNsrOZy",
"hfcjrAqbF3YhFCVBTkps9RZxib6uOV1HUtHfPEcZF4C+ciHQElCmCsmcMvhGNrnw+l5PJ5OHZUH/Aruf",
"PFDBQVrO9pOHO7X0/1q+AWPJJt8vHoJgzvbjv3meOrSxxBbeGf/WkOEp/tfkAMGk9P8kOP9TWLvfj7CG",
"+4JrYHj6J+YM13K+1CrU8g6odSoam5NOyZUxfClqPzjfO4FgvO2y2Dg97lwCLDh1XC5yrVYajMEjnBEu",
"gEXKD/YF5fNAheO0KfnC5QoRGQHd5EzGQbBFxoUFnTDnrX/hQTVU5YCWxABDSiK/EQWSoC0RRbCOW9ic",
"9P9btzeIdkaVVhKtyc795nIFxh3ge85Vb0Y50WQDbn/XbJBswYiFHlYTY5F7jVR2EDjuIZy2vaK4QRnX",
"A4XtE2QbENIq8264L0DvXHIohDUuvJcV/4CllcWkL/Wk+B6D1YOD5xiiShQb2XE1VZLxKoMMZMVlvWc/",
"KikqyaYHLfem8kJgJZfOHf2gBbqmpYV3qDCQFaJKk4GHDX53hDZZ3PJvZMMo8kh9lBOOv4x9eCThBPOz",
"GpRaUyPzCDBmYdfE6V9pIBZ09ZMKZYAtuLSgt0TgEVY5yPi3gMwuuss0X61Tz7mkomDgc374Xyqtzdh/",
"wRiy8gg36ZNi/2fJ7wtAnLlakXHQoU5Ky+3uNNd7eP5eLS+VEECtShA9fos2ioFIJJVQ1Rbl011XynVY",
"gaoV/tyx6IFpId5y6RE8XguoWxOitK3tyU0I5/kOQz7n7KQhhVvzgwwJ5xloyAdu7P8K0KX65mHuDy+a",
"Z3C7XOoqFwwuoE7T7mTSqdSm2B4kdE5KqOsOF1vQpjfbhDWoXNOoP86ATOkNsXiKubSvfzl4zyWFVSj4",
"VMmMr7rSL/3zQoeW0jcxseRO2hieG47LEa43s8etDmseY3UFzeIkHaqVZVGfXZlzSk/1ewCF5m5hsgH2",
"IkZtPnRc1UutQ1bqdANnQC/h6xHYfhaH+t29npqDsbfg+5Wuu5iixQakTTaBs6uKhtUy3684mpIV4dKk",
"L3beB/1k/3+C4U5uYSDg4horuRpE+pYjYmva5zjhn3Dn7TrI909dI259H9yyAZIOaZ0xCOw/TcmEprpP",
"OVCXbIxX53Z2EkTVfL3/dPP74vqP+e2by/nNLR7hy5vf59d/zBdvP3/4kGyOvN5DcTwvYYd9rJW0H5mn",
"K6H0zHx9OmAryeJJArdbqPf+epmpruY3H2d1GDXt8f5HN5quwdjSVgN6y2mo2JZbP8pIrXvzceaa/Aod",
"/Gp8Mb7w05UcJMk5nuLX44vxa9e/ELv2Rk3Ctc1jrUJSaMW95Ja7PBwPW3KtKBh/IMcPr37GXPEKc4Aw",
"OcABBDD2N8V2ZUK2IL0WkueCh3nO5M4ENoXsNmygUo0l9s2YsroA/yDEsLfxl4tXT6b8cHnwittdXjw8",
"A4ZMQZ2fskKI3dgh8evFRcrFWyL8vMw7Cy0Vq1b/mrwcIKlsNd9ypzDFZkNch1X5/zCEcZQhK+OSQTlL",
"CmTCX9zGEv3JA2d7p2kFCQZcroH+FeCnhdYu8/sBkwuZxrCnSYXD6Ao84ar5CJ7+mWp5DvXlIJC7l46q",
"eITDlTz0C024RxF07XT7pUOFiyfmYTVV7afDmhi0BJCoHsL1g3t9GOL1YuzxqCLRIzE+gfKdWv6HVjcN",
"0x/roY8yiPheyB2CxpeeJr5hceMi+zzxnrh/vqSgfx+7KVx+Hxv5Tbi9RESaeESIx/45hvvxKL8Fqzls",
"PfomdBi0qRItd4hbg2ZXXTK8Axsf47fdjJ0X9E1NViFdnufl54FGCJwkCAv54nh+PyzvzQLvwLY54QCa",
"XZ1kxgjnhe3rhkxI6dy33S3xX7ld+8xQ29CmQZDRygmPYUFRj0WeiAPPm57KxnlQejoJfVG2p8/VRAwg",
"WbDnu3JP1HWfzDeiOZmKv/v4Ly9Ucwuaky7h4knYM4Z4rCYR4W/aFgyHKf5u1AdW6PV7QfJzh/vaCRU0",
"flcj6gfU/HDz8YHu4jHXassZsP6ID9vDgO954iue8Lykuh9QecZ6X+PRD2oUaHV1Z+B6zK7uK8g10BLr",
"usIHwI9U9npbhfEZ6bwe57BKyLNW8xQ+tebnyaMnQrN2HiKnoByd0ZadBO0d+HS1O78PqyH7efqv8vtE",
"X4Q2G64fk5JDc1bD1GjKunmZWLoe1o9F6XlAH/aYiP0JGq94VDmoMPQmiSfptNKN05kJfGKryXyyWF9/",
"A1qEDO4n77rwo3PSW7ebf5bSmtVBaGq+lyLViPtlEyT+4jGcJU+tv/yikEhSc4+j/4LwOMp5QacJ5/aA",
"3qaR/qgVK2g9dAbX6hda4CleW5ub6WRCcj4uP3ePqdpMtq+wg6/U1pZ3UxHOhD8GBOaIc+hVS7o0zrgf",
"DZPSuJJEwlJXkqEyw1AtEtacpu2/7P8JAAD//02yIGRmKQAA",
"H4sIAAAAAAAC/+RaX2/bOBL/KgTvHg073e6T39okLVz0Nr3UPSywKAyaHNnMUqRCUm69gb/7gaQkUxZl",
"y/mz6KJPiS1yhjO/3/zhyA+YqrxQEqQ1ePqADV1DTvy/198Lpe0VWMKF/4KBoZoXliuJp/gT2QpFGMqU",
"RuCXIqv5agUaaTCFkgbGeIQLrQrQloMXcaeWC866wuZrQHdqiThDGgSxfAPIKmTXUMl2ouy2ADzFxmou",
"V3g3wqq0RWkXQlES5KTE1k8Rl+jbmtN1JBX9xQuUcQHoGxcCLQFlqpTMKYPvJC+E1/d6Opk8LEv6J9jd",
"5IEKDtJytps83Kml/2t5DsaSvNgtHoJgznbjv3iROrSxxJbeGf/WkOEp/tdkD8Gk8v8kOP9zWLvbjbCG",
"+5JrYHj6B+YMN3K+NirU8g6odSpam5NOKZQxfCkaPzjfO4FgvO2yzJ0edy4BFpw6LheFVisNxuARzggX",
"wCLle/uC8nmgwnHaVHzhcoWIjIBucybjINgi48KCTpjzzj/woBqqCkBLYoAhJZHfiAJJ0IaIMljHLeQn",
"/f/O7Q2inVGVlURrsnWfuVyBcQd4zLmazaggmuTg9nfNBskWjFjoYTUxFrnHSGV7geMewmnbK4oblHE9",
"UNguQbYBIa0y74b7EvTWJYdSWOPCe1nzD1haWUz6Sk+K7zFYPTh4jiGqRJnLjqupkozXGWQgKy6bPbtR",
"RVFJ8h603JPaC4GVXDp39IMW6JqWFp6h0kBWijpNBh62+N0R2mbxgX8jG0aRR5qjnHD8ZezDIwknmJ81",
"oDSaWplHgDELuyZO/0oDsaDrj1QoA2zBpQW9IQKPsCpAxp8FZHbRXab5ap36nksqSgY+54f/Umltxv4D",
"xpCVR7hNnxT7v0h+XwLizNWKjIMOdVJabrenud7D8w9qeamEAGpVgujxU5QrBqKbVAh1pXWxAW16oQpr",
"ULWmpi2tRTuhmdI5sXiKubSvf9mb47y6ChnT49xD4PDMxYCtkkRL+qAEHVvraZhK08dy0+yqNo0Yoygn",
"Fpg7SzIchSuN9rjfwpqn+i3nkudlvqACiAR9XGW1uNHpi+oaEFO0zEFaVEk5U7eF7/Zxit1OV9XMQJ2p",
"xqYCraFQv096TtyBa3TI+1PBFQjVz120j6rG9A6N25HXXxyqBiUIdsuSDPTFs5fMobRyhuya2NDMFqoo",
"nR/84ULRPZ14qvTfKDvlqC9Fuq+Iu7zSrQn1N9760yenny7SO0z6yI39bwm6wr/Nhvv9g/ap3C4HfLVg",
"MChO0/ZkG1SrTTE/SOic9PG89XE2EDKqZMZXXemX/vtSh0tuDU8juZNIhncrx+U8viSeY3UNzeIkHeqV",
"VS6cXZlzmuH68wAKzd3CZOXyIjrFpuOqXmpd+k63S7DzoJfw7Qhs/xSH+t29npqDsbfgb1Bdd9U5MVkt",
"921fkzrdDcrRlKwIl8YeKb+9ZP9fguFObmkg4OKuenL1mJ4otubwHCf8E6ZwXQf5G13XiFvfJBzYAGZA",
"2xAE9p+mYkJb3ecCqEs2JtSTbQGdBFFfBz98vvltcf37/PbN5fzmFo/w5c1v8+vf54t3Xz5+TF7XvN59",
"d3Jewg772EHSfmKeroXSM/P16YCtJYtnCdxuod75gVemuprffJo1YdS2x/sf3Wi6BmMrWw3oDaehYltu",
"/XA1te7Npxke4QYd/Gp8Mb7w894CJCk4nuLX44vxa9dAErv2Rk3u1HIShkkebxUSw0HsS27dPc/EI+BC",
"KwrGH8pxxB9hxlwBC9PJMM/EAQgw9q1i2yopW5BeCykKwcOUeXJnAqNChhs25q2Hpbt2XFldgv8ixLG3",
"85eLV8+mfD/S8IrbvrpujfSBIVNS56esFGI7dmj8enGRcvGGCD/F985CS8Xq1b8mRxZIKltP3d0pTJnn",
"xHVZtf/3o2FHG7IyLiFUE+5AKPzVbYwYMHngbOe0rSDBgss10D8DBWiptasAfvTtQqc1hm7TYT9UB0+8",
"enKLp38cHy/sBXL30FEWj6r7YOgb2pCPIvgO0+7XDh0unpmL9fuefkqsiUFLAIma1wP9AF/vXy/04uzx",
"qKPRIzEegLSDeELjaVgS7FuwmsMGDCLIhIJD25c6tNwibg2aXXUxfw82vsq93X5QyxkbAr9TMbtq0mH7",
"CvlD06A1ZEyw4EPLeSzQ5XiI75f3kuA9WES6wMyuIirEJ9sTwhcBuu4riybENPf914GCb9yufbvcWHFI",
"gCDjkRzYp4C2WtcXernPSIbnL02JMc+g+nSSA2XVrrxUQRnAtmDPIeFOUs3lnqgLO5lwRHtSEb+Z9O8G",
"qeYWNCdd3sWTkReM9VhNItTfHFowHKb4zWYfWKH36wXJ30PvGyfU0PhdrfBP9nrhLu1QaO7DId5dPBZa",
"bTgD1h/4YXsY+LxMfMU3/h+p8QuoUH+wJwZpu8x7iTEe/aBGgdb0cgxcr9HVfQWFBlph3ZT4APiR0t5s",
"qzE+I50313tWC3nRsp7Cp9H8Mnn0RGg2zkPkFJSjM/qyk6C9B5+utm+351bgBjJdKf7xG7FqXt0Xoe3O",
"6+9JyaFLa2BqdWfdvDy8LYvS84B27CkR+w9ovOLR1aDC0JsknqXTSjdOZybwia0ntcliff0daBkyuJ/E",
"6tKPUklv3W7/cOpgbgOhqXksReqR549NkHgCPpwlz62/mjAnktTc41i9dn4K5byg04Rze0Bv0kh/0oqV",
"tBlCgmv1Sy3wFK+tLcx0MiEFH1c/+RxTlU82r7CDr9J2KO+mJpwJP1cF5oiz71UrurTOuBsNk9K6kkTC",
"UleSoTLDcCUS1p6q7L7u/h8AAP//+pSl8AgsAAA=",
}
// GetSwagger returns the content of the embedded swagger specification file
api/queryService/jobcollector.go
+32 -7
View File
@@ -1,19 +1,44 @@
package queryservice
import (
"fmt"
"net/http"
"github.com/google/uuid"
"github.com/labstack/echo/v4"
)
func (s *Controllers) GetJobCollectorById(ctx echo.Context, id string) error {
return ctx.JSON(http.StatusOK, map[string]string{"status": "export triggered"})
func (s *Controllers) GetJobCollectorByJobId(ctx echo.Context, jobId string) error {
uid, err := uuid.Parse(jobId)
if err != nil {
return echo.NewHTTPError(http.StatusBadRequest, "Invalid ID")
}
coll, err := s.svc.JobCollector.GetByJobID(ctx.Request().Context(), uid)
if err != nil {
return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Unable to get collector: %s", err))
}
fields := make([]JobCollectorField, len(coll.Fields))
index := 0
for name, id := range coll.Fields {
fields[index] = JobCollectorField{
Name: name,
QueryId: id.String(),
}
index++
}
return ctx.JSON(http.StatusOK, JobCollector{
JobId: coll.JobID.String(),
MinimumCleanerVersion: coll.MinCleanVersion,
MinimumTextVersion: coll.MinTextVersion,
ActiveVersion: coll.ActiveVersion,
LatestVersion: coll.LatestVersion,
Fields: fields,
})
}
func (s *Controllers) CreateJobCollector(ctx echo.Context) error {
return ctx.JSON(http.StatusOK, map[string]string{"status": "export triggered"})
}
func (s *Controllers) UpdateJobCollector(ctx echo.Context, id string) error {
func (s *Controllers) UpdateJobCollectorByJobId(ctx echo.Context, jobId string) error {
return ctx.JSON(http.StatusOK, map[string]string{"status": "export triggered"})
}
api/queryService/jobcollector_test.go
+46 -20
View File
@@ -1,32 +1,23 @@
package queryservice_test
import (
"encoding/json"
"net/http"
"net/http/httptest"
queryservice "queryorchestration/api/queryService"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/job/collector"
"strings"
"testing"
"github.com/go-playground/validator/v10"
"github.com/google/uuid"
"github.com/labstack/echo/v4"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
)
func TestCreateJobCollector(t *testing.T) {
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(""))
rec := httptest.NewRecorder()
ctx := e.NewContext(req, rec)
cons := queryservice.NewControllers(validator.New(), &queryservice.Services{})
err := cons.CreateJobCollector(ctx)
assert.Nil(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
assert.NotEmpty(t, rec.Body.String())
}
func TestUpdateJobCollector(t *testing.T) {
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(""))
@@ -37,24 +28,59 @@ func TestUpdateJobCollector(t *testing.T) {
id := uuid.New()
err := cons.UpdateJobCollector(ctx, id.String())
err := cons.UpdateJobCollectorByJobId(ctx, id.String())
assert.Nil(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
assert.NotEmpty(t, rec.Body.String())
}
func TestGetJobCollector(t *testing.T) {
func TestGetJobCollectorByJobId(t *testing.T) {
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(""))
rec := httptest.NewRecorder()
ctx := e.NewContext(req, rec)
cons := queryservice.NewControllers(validator.New(), &queryservice.Services{})
svc := collector.New(db)
cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
JobCollector: svc,
})
id := uuid.New()
coll := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
MinCleanVersion: int32(1),
MinTextVersion: int32(2),
}
err := cons.GetJobCollectorById(ctx, id.String())
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(coll.JobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(coll.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")),
)
err = cons.GetJobCollectorByJobId(ctx, coll.JobID.String())
assert.Nil(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
assert.NotEmpty(t, rec.Body.String())
var res queryservice.JobCollector
err = json.Unmarshal(rec.Body.Bytes(), &res)
assert.Nil(t, err)
assert.EqualExportedValues(t, queryservice.JobCollector{
JobId: coll.JobID.String(),
MinimumCleanerVersion: coll.MinCleanVersion,
MinimumTextVersion: coll.MinTextVersion,
ActiveVersion: int32(1),
LatestVersion: int32(2),
Fields: []queryservice.JobCollectorField{},
}, res)
}
cmd/queryRunner/main.go
+6 -1
View File
@@ -3,6 +3,7 @@ package main
import (
"context"
controllers "queryorchestration/api/queryRunner"
"queryorchestration/internal/job/collector"
"queryorchestration/internal/query/document"
"queryorchestration/internal/server"
"queryorchestration/internal/server/queue"
@@ -14,7 +15,11 @@ func main() {
ctx := context.Background()
queryrunner := func(cfg *server.Config) queue.Controller {
svc := document.New(cfg.Database)
coll := collector.New(cfg.Database)
svc := document.New(cfg.Database, &document.Services{
Collector: coll,
})
return controllers.NewQueryRunner(svc, cfg.Validator)
}
database/migrations/20241219151404_create_activequeryrequirements_view.down.sql
-1
View File
@@ -1 +0,0 @@
DROP VIEW activeQueryRequirements;
database/migrations/20241219151404_create_activequeryrequirements_view.up.sql
-6
View File
@@ -1,6 +0,0 @@
CREATE VIEW activeQueryRequirements AS
SELECT q.id, q.type, q.activeVersion, rq.requiredQueryId
FROM queries q
JOIN requiredQueries rq ON q.id = rq.queryId
AND q.activeVersion >= rq.addedVersion
AND q.activeVersion < COALESCE(rq.removedVersion, q.activeVersion + 1);
database/migrations/20241223113819_create_queryconfig_table.down.sql → database/migrations/20241219161629_create_queryconfig_table.down.sql
View File
database/migrations/20241223113819_create_queryconfig_table.up.sql → database/migrations/20241219161629_create_queryconfig_table.up.sql
View File
database/migrations/20250108174833_create_fullactivequery_view.down.sql → database/migrations/20241219161630_create_fullactivequery_view.down.sql
View File
database/migrations/20250108174833_create_fullactivequery_view.up.sql → database/migrations/20241219161630_create_fullactivequery_view.up.sql
+4 -4
View File
@@ -2,9 +2,9 @@ CREATE VIEW fullActiveQueries AS
SELECT DISTINCT q.id, q.type, q.activeVersion, q.latestVersion, coalesce(c.config, null) as config, ARRAY_AGG(DISTINCT r.requiredQueryId)::uuid[] as requiredIds
FROM queries AS q
LEFT JOIN queryConfigs AS c ON q.id = c.queryId
and c.addedVersion >= q.activeVersion
and COALESCE(c.removedVersion, q.activeVersion - 1) < q.activeVersion
and q.activeVersion >= c.addedVersion
and q.activeVersion < COALESCE(c.removedVersion, q.activeVersion + 1)
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
and r.addedVersion >= q.activeVersion
and COALESCE(r.removedVersion, q.activeVersion - 1) < q.activeVersion
and q.activeVersion >= r.addedVersion
and q.activeVersion < COALESCE(r.removedVersion, q.activeVersion + 1)
GROUP BY q.id, q.type, q.activeversion, q.latestversion, c.config;
database/migrations/20241219161635_create_activecollectorqueries_view.down.sql
-1
View File
@@ -1 +0,0 @@
DROP VIEW activeCollectorQueries;
database/migrations/20241219161635_create_activecollectorqueries_view.up.sql
-6
View File
@@ -1,6 +0,0 @@
CREATE VIEW activeCollectorQueries AS
SELECT c.id as collectorId, c.activeVersion, cq.queryId
FROM collectors c
JOIN collectorQueries cq ON c.id = cq.collectorId
AND c.activeVersion >= cq.addedVersion
AND c.activeVersion < COALESCE(cq.removedVersion, c.activeVersion + 1);
database/migrations/20241219161635_create_activecollectorswithrequiredids_view.down.sql
+1
View File
@@ -0,0 +1 @@
DROP VIEW activeCollectorsWithRequiredIDs;
database/migrations/20241219161635_create_activecollectorswithrequiredids_view.up.sql
+7
View File
@@ -0,0 +1,7 @@
CREATE VIEW activeCollectorsWithRequiredIDs AS
SELECT DISTINCT c.id, c.activeVersion, array_agg(q.queryId) as queryIds
FROM collectors as c
LEFT JOIN collectorQueries as q ON c.id = q.collectorId
AND c.activeVersion >= q.addedVersion
and c.activeVersion < COALESCE(q.removedVersion, c.activeVersion + 1)
GROUP BY c.id, c.activeVersion;
database/migrations/20241219161636_create_collectorquerydep_view.up.sql
+9 -8
View File
@@ -1,13 +1,14 @@
CREATE VIEW collectorQueryDependencyTree (collectorId, queryId, type, requiredQueryId, queryVersion) AS
CREATE VIEW collectorQueryDependencyTree AS
WITH RECURSIVE collectorQueryDependencyTree AS (
SELECT cq.collectorId, aqc.id, aqc.type, aqc.requiredQueryId, aqc.activeVersion as queryVersion
FROM activeQueryRequirements aqc
JOIN activeCollectorQueries cq ON cq.queryId = aqc.id
SELECT cq.id, aqc.id as queryId, aqc.type, aqc.requiredIds, aqc.activeVersion
FROM fullActiveQueries as aqc
JOIN activeCollectorsWithRequiredIDs as cq ON aqc.id = ANY(cq.queryIds)
UNION ALL
SELECT acq.collectorId, q.id as queryId, q.type, q.requiredQueryId, q.activeVersion as queryVersion
FROM activeQueryRequirements q
JOIN collectorQueryDependencyTree acq on q.id = acq.requiredQueryId
SELECT acq.id, q.id as queryId, q.type, q.requiredIds, q.activeVersion
FROM fullActiveQueries as q
JOIN collectorQueryDependencyTree as acq on q.id = ANY(acq.requiredIds)
)
SELECT DISTINCT * FROM collectorQueryDependencyTree;
SELECT id as collectorId, queryId, type, activeVersion as queryVersion, requiredIds::uuid[]
FROM collectorQueryDependencyTree;
database/migrations/20250120165410_create_fullactivecollector_view.down.sql
+1
View File
@@ -0,0 +1 @@
DROP VIEW fullActiveCollectors;
database/migrations/20250120165410_create_fullactivecollector_view.up.sql
+8
View File
@@ -0,0 +1,8 @@
CREATE VIEW fullActiveCollectors AS
SELECT DISTINCT c.id, c.jobId, c.minCleanVersion, c.minTextVersion, c.activeVersion, c.latestVersion,
jsonb_object_agg(q.name, q.queryId) FILTER (WHERE q.name is not null) AS fields
FROM collectors AS c
LEFT JOIN collectorQueries AS q ON c.id = q.collectorId
AND c.activeVersion >= q.addedVersion
and c.activeVersion < COALESCE(q.removedVersion, c.activeVersion + 1)
GROUP BY c.id, c.jobId, c.minCleanVersion, c.minTextVersion;
database/queries/collector.sql
+13 -7
View File
@@ -1,8 +1,14 @@
-- name: GetCollectorQueries :many
SELECT collectorId, queryId, type, queryVersion, ARRAY_AGG(requiredQueryId) AS requiredIds
FROM collectorQueryDependencyTree
WHERE collectorId = $1
GROUP BY queryId, collectorId, type, queryVersion;
-- name: ListCollectorQueries :many
SELECT * FROM collectorQueryDependencyTree WHERE collectorId = $1;
-- name: GetCollectorFromJobID :one
SELECT id, jobId, minCleanVersion, minTextVersion FROM collectors WHERE jobId = $1 LIMIT 1;
-- name: GetCollectorByJobID :one
SELECT * FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1;
-- name: GetCollector :one
SELECT * FROM fullActiveCollectors WHERE id = $1 LIMIT 1;
-- name: CreateCollector :one
INSERT INTO collectors (jobId, minCleanVersion, minTextVersion) VALUES ($1, $2, $3) RETURNING id;
-- name: AddCollectorQuery :exec
INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4);
database/queries/query.sql
+2 -2
View File
@@ -10,10 +10,10 @@ SELECT EXISTS (
);
-- name: GetQuery :one
SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries WHERE id = $1;
SELECT * FROM fullActiveQueries WHERE id = $1;
-- name: ListQueries :many
SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries;
SELECT * FROM fullActiveQueries;
-- name: CreateQuery :one
INSERT INTO queries (type) VALUES ($1) RETURNING id;
internal/database/repository/collector.sql.go
+77 -29
View File
@@ -11,62 +11,110 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
const getCollectorFromJobID = `-- name: GetCollectorFromJobID :one
SELECT id, jobId, minCleanVersion, minTextVersion FROM collectors WHERE jobId = $1 LIMIT 1
const addCollectorQuery = `-- name: AddCollectorQuery :exec
INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
`
type GetCollectorFromJobIDRow struct {
ID pgtype.UUID `db:"id"`
type AddCollectorQueryParams struct {
Collectorid pgtype.UUID `db:"collectorid"`
Name string `db:"name"`
Queryid pgtype.UUID `db:"queryid"`
Addedversion int32 `db:"addedversion"`
}
// AddCollectorQuery
//
// INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
func (q *Queries) AddCollectorQuery(ctx context.Context, arg *AddCollectorQueryParams) error {
_, err := q.db.Exec(ctx, addCollectorQuery,
arg.Collectorid,
arg.Name,
arg.Queryid,
arg.Addedversion,
)
return err
}
const createCollector = `-- name: CreateCollector :one
INSERT INTO collectors (jobId, minCleanVersion, minTextVersion) VALUES ($1, $2, $3) RETURNING id
`
type CreateCollectorParams struct {
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
}
// GetCollectorFromJobID
// CreateCollector
//
// SELECT id, jobId, minCleanVersion, minTextVersion FROM collectors WHERE jobId = $1 LIMIT 1
func (q *Queries) GetCollectorFromJobID(ctx context.Context, jobid pgtype.UUID) (*GetCollectorFromJobIDRow, error) {
row := q.db.QueryRow(ctx, getCollectorFromJobID, jobid)
var i GetCollectorFromJobIDRow
// INSERT INTO collectors (jobId, minCleanVersion, minTextVersion) VALUES ($1, $2, $3) RETURNING id
func (q *Queries) CreateCollector(ctx context.Context, arg *CreateCollectorParams) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, createCollector, arg.Jobid, arg.Mincleanversion, arg.Mintextversion)
var id pgtype.UUID
err := row.Scan(&id)
return id, err
}
const getCollector = `-- name: GetCollector :one
SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE id = $1 LIMIT 1
`
// GetCollector
//
// SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE id = $1 LIMIT 1
func (q *Queries) GetCollector(ctx context.Context, id pgtype.UUID) (*Fullactivecollector, error) {
row := q.db.QueryRow(ctx, getCollector, id)
var i Fullactivecollector
err := row.Scan(
&i.ID,
&i.Jobid,
&i.Mincleanversion,
&i.Mintextversion,
&i.Activeversion,
&i.Latestversion,
&i.Fields,
)
return &i, err
}
const getCollectorQueries = `-- name: GetCollectorQueries :many
SELECT collectorId, queryId, type, queryVersion, ARRAY_AGG(requiredQueryId) AS requiredIds
FROM collectorQueryDependencyTree
WHERE collectorId = $1
GROUP BY queryId, collectorId, type, queryVersion
const getCollectorByJobID = `-- name: GetCollectorByJobID :one
SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
`
type GetCollectorQueriesRow struct {
Collectorid pgtype.UUID `db:"collectorid"`
Queryid pgtype.UUID `db:"queryid"`
Type NullQuerytype `db:"type"`
Queryversion *int32 `db:"queryversion"`
Requiredids []pgtype.UUID `db:"requiredids"`
// GetCollectorByJobID
//
// SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
func (q *Queries) GetCollectorByJobID(ctx context.Context, jobid pgtype.UUID) (*Fullactivecollector, error) {
row := q.db.QueryRow(ctx, getCollectorByJobID, jobid)
var i Fullactivecollector
err := row.Scan(
&i.ID,
&i.Jobid,
&i.Mincleanversion,
&i.Mintextversion,
&i.Activeversion,
&i.Latestversion,
&i.Fields,
)
return &i, err
}
// GetCollectorQueries
const listCollectorQueries = `-- name: ListCollectorQueries :many
SELECT collectorid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
`
// ListCollectorQueries
//
// SELECT collectorId, queryId, type, queryVersion, ARRAY_AGG(requiredQueryId) AS requiredIds
// FROM collectorQueryDependencyTree
// WHERE collectorId = $1
// GROUP BY queryId, collectorId, type, queryVersion
func (q *Queries) GetCollectorQueries(ctx context.Context, collectorid pgtype.UUID) ([]*GetCollectorQueriesRow, error) {
rows, err := q.db.Query(ctx, getCollectorQueries, collectorid)
// SELECT collectorid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.UUID) ([]*Collectorquerydependencytree, error) {
rows, err := q.db.Query(ctx, listCollectorQueries, collectorid)
if err != nil {
return nil, err
}
defer rows.Close()
items := []*GetCollectorQueriesRow{}
items := []*Collectorquerydependencytree{}
for rows.Next() {
var i GetCollectorQueriesRow
var i Collectorquerydependencytree
if err := rows.Scan(
&i.Collectorid,
&i.Queryid,
internal/database/repository/collector_test.go
+82 -8
View File
@@ -2,6 +2,7 @@ package repository_test
import (
"context"
"fmt"
"os"
"path"
"queryorchestration/internal/database"
@@ -10,6 +11,7 @@ import (
"testing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
)
@@ -23,15 +25,87 @@ func TestCollector(t *testing.T) {
queries := repository.New(db.Pool)
collectorID := database.MustToDBUUID(uuid.New())
collectorQueries, err := queries.GetCollectorQueries(ctx, collectorID)
contextId, err := queries.CreateQuery(ctx, repository.QuerytypeContextFull)
assert.Nil(t, err)
jsonId, err := queries.CreateQuery(ctx, repository.QuerytypeJsonExtractor)
assert.Nil(t, err)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: jsonId,
Requiredqueryid: contextId,
Addedversion: 1,
})
assert.Nil(t, err)
assert.Len(t, collectorQueries, 0)
assert.ElementsMatch(t, []repository.GetCollectorQueriesRow{}, collectorQueries)
jobID := database.MustToDBUUID(uuid.New())
jobId := database.MustToDBUUID(uuid.New())
minCleanVersion := int32(2)
minTextVersion := int32(4)
_, err = queries.GetCollectorFromJobID(ctx, jobID)
assert.NotNil(t, err)
collId, err := queries.CreateCollector(ctx, &repository.CreateCollectorParams{
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
})
assert.Nil(t, err)
coll, err := queries.GetCollector(ctx, collId)
assert.Nil(t, err)
assert.EqualExportedValues(t, repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
Activeversion: 1,
Latestversion: 1,
}, *coll)
coll, err = queries.GetCollectorByJobID(ctx, jobId)
assert.Nil(t, err)
assert.EqualExportedValues(t, repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
Activeversion: 1,
Latestversion: 1,
}, *coll)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collId,
Queryid: jsonId,
Addedversion: 1,
Name: "example_key",
})
assert.Nil(t, err)
coll, err = queries.GetCollector(ctx, collId)
assert.Nil(t, err)
assert.EqualExportedValues(t, repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
Activeversion: 1,
Latestversion: 1,
Fields: []byte(fmt.Sprintf("{\"example_key\": \"%s\"}", database.MustToUUID(jsonId).String())),
}, *coll)
qs, err := queries.ListCollectorQueries(ctx, collId)
assert.Nil(t, err)
assert.Len(t, qs, 2)
assert.ElementsMatch(t, []*repository.Collectorquerydependencytree{
{
Collectorid: collId,
Queryid: jsonId,
Queryversion: 1,
Type: repository.QuerytypeJsonExtractor,
Requiredids: []pgtype.UUID{contextId},
},
{
Collectorid: collId,
Queryid: contextId,
Queryversion: 1,
Type: repository.QuerytypeContextFull,
Requiredids: []pgtype.UUID{database.MustToDBUUID(uuid.Nil)},
},
}, qs)
}
internal/database/repository/models.go
+17 -14
View File
@@ -62,17 +62,10 @@ func (e Querytype) Valid() bool {
return false
}
type Activecollectorquery struct {
Collectorid pgtype.UUID `db:"collectorid"`
type Activecollectorswithrequiredid struct {
ID pgtype.UUID `db:"id"`
Activeversion int32 `db:"activeversion"`
Queryid pgtype.UUID `db:"queryid"`
}
type Activequeryrequirement struct {
ID pgtype.UUID `db:"id"`
Type Querytype `db:"type"`
Activeversion int32 `db:"activeversion"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
Queryids interface{} `db:"queryids"`
}
type Collector struct {
@@ -94,11 +87,21 @@ type Collectorquery struct {
}
type Collectorquerydependencytree struct {
Collectorid pgtype.UUID `db:"collectorid"`
Collectorid pgtype.UUID `db:"collectorid"`
Queryid pgtype.UUID `db:"queryid"`
Type Querytype `db:"type"`
Queryversion int32 `db:"queryversion"`
Requiredids []pgtype.UUID `db:"requiredids"`
}
type Fullactivecollector struct {
ID pgtype.UUID `db:"id"`
Type Querytype `db:"type"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
Queryversion int32 `db:"queryversion"`
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
Activeversion int32 `db:"activeversion"`
Latestversion int32 `db:"latestversion"`
Fields []byte `db:"fields"`
}
type Fullactivequery struct {
internal/database/repository/query.sql.go
+4 -4
View File
@@ -92,12 +92,12 @@ func (q *Queries) DeprecateQuery(ctx context.Context, queryid pgtype.UUID) error
}
const getQuery = `-- name: GetQuery :one
SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries WHERE id = $1
SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = $1
`
// GetQuery
//
// SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries WHERE id = $1
// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = $1
func (q *Queries) GetQuery(ctx context.Context, id pgtype.UUID) (*Fullactivequery, error) {
row := q.db.QueryRow(ctx, getQuery, id)
var i Fullactivequery
@@ -155,12 +155,12 @@ func (q *Queries) IsQueryDeprecated(ctx context.Context, queryid pgtype.UUID) (b
}
const listQueries = `-- name: ListQueries :many
SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries
SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries
`
// ListQueries
//
// SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries
// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries
func (q *Queries) ListQueries(ctx context.Context) ([]*Fullactivequery, error) {
rows, err := q.db.Query(ctx, listQueries)
if err != nil {
internal/database/repository/query_test.go
+18
View File
@@ -102,6 +102,24 @@ func TestQueries(t *testing.T) {
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Latestversion: 2,
Config: jsonConfig,
Requiredids: []pgtype.UUID{contextQueryID},
}, *jsonQuery)
err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{
Activeversion: 2,
Latestversion: 2,
ID: jsonQueryID,
})
assert.Nil(t, err)
jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
assert.Nil(t, err)
assert.EqualExportedValues(t, repository.Fullactivequery{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 2,
Latestversion: 2,
Config: nil,
Requiredids: []pgtype.UUID{database.MustToDBUUID(uuid.Nil)},
}, *jsonQuery)
internal/job/collector/collector.go
-43
View File
@@ -1,43 +0,0 @@
package collector
import (
"context"
"queryorchestration/internal/database"
"github.com/google/uuid"
)
type Collector struct {
ID uuid.UUID
MinCleanVersion int32
MinTextVersion int32
db *database.Connection
}
func NewByJobId(ctx context.Context, db *database.Connection, jobID uuid.UUID) (*Collector, error) {
collector := Collector{
db: db,
}
err := collector.getByJobID(ctx, jobID)
if err != nil {
return nil, err
}
return &collector, nil
}
func (c *Collector) getByJobID(ctx context.Context, jobID uuid.UUID) error {
dbJobID := database.MustToDBUUID(jobID)
dbCollector, err := c.db.Queries.GetCollectorFromJobID(ctx, dbJobID)
if err != nil {
return err
}
c.ID = database.MustToUUID(dbCollector.ID)
c.MinCleanVersion = dbCollector.Mincleanversion
c.MinTextVersion = dbCollector.Mintextversion
return nil
}
internal/job/collector/get.go
+26
View File
@@ -0,0 +1,26 @@
package collector
import (
"context"
"queryorchestration/internal/database"
"github.com/google/uuid"
)
func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Collector, error) {
dbColl, err := s.db.Queries.GetCollector(ctx, database.MustToDBUUID(id))
if err != nil {
return nil, err
}
return parseDBCollector(dbColl)
}
func (s *Service) GetByJobID(ctx context.Context, jobID uuid.UUID) (*Collector, error) {
dbColl, err := s.db.Queries.GetCollectorByJobID(ctx, database.MustToDBUUID(jobID))
if err != nil {
return nil, err
}
return parseDBCollector(dbColl)
}
internal/job/collector/get_test.go
+86
View File
@@ -0,0 +1,86 @@
package collector_test
import (
"context"
"fmt"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/job/collector"
"testing"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
)
func TestGet(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := collector.New(db)
ogc := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
MinCleanVersion: 2,
MinTextVersion: 4,
Fields: map[string]uuid.UUID{
"example_key": uuid.New(),
},
}
pool.ExpectQuery("name: GetCollector :one").WithArgs(database.MustToDBUUID(ogc.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(ogc.ID), database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))),
)
coll, err := svc.Get(ctx, ogc.ID)
assert.Nil(t, err)
assert.EqualExportedValues(t, ogc, *coll)
}
func TestGetByJobID(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := collector.New(db)
ogc := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
MinCleanVersion: 2,
MinTextVersion: 4,
Fields: map[string]uuid.UUID{
"example_key": uuid.New(),
},
}
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(ogc.JobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(ogc.ID), database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))),
)
coll, err := svc.GetByJobID(ctx, ogc.JobID)
assert.Nil(t, err)
assert.EqualExportedValues(t, ogc, *coll)
}
internal/job/collector/parse.go
+37
View File
@@ -0,0 +1,37 @@
package collector
import (
"encoding/json"
"fmt"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"github.com/google/uuid"
)
func parseDBCollector(c *repository.Fullactivecollector) (*Collector, error) {
if c == nil {
return nil, nil
}
var fields map[string]uuid.UUID
if len(c.Fields) > 0 {
err := json.Unmarshal(c.Fields, &fields)
if err != nil {
return nil, fmt.Errorf("error unmarshalling: %s", err)
}
} else {
fields = map[string]uuid.UUID{}
}
return &Collector{
ID: database.MustToUUID(c.ID),
JobID: database.MustToUUID(c.Jobid),
MinCleanVersion: c.Mincleanversion,
MinTextVersion: c.Mintextversion,
ActiveVersion: c.Activeversion,
LatestVersion: c.Latestversion,
Fields: fields,
}, nil
}
internal/job/collector/parse_test.go
+36
View File
@@ -0,0 +1,36 @@
package collector
import (
"fmt"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
func TestParseDBCollector(t *testing.T) {
c, err := parseDBCollector(nil)
assert.Nil(t, err)
assert.Nil(t, c)
ogc := Collector{
ID: uuid.New(),
JobID: uuid.New(),
MinCleanVersion: 1,
MinTextVersion: 2,
Fields: map[string]uuid.UUID{
"example_key": uuid.New(),
},
}
c, err = parseDBCollector(&repository.Fullactivecollector{
ID: database.MustToDBUUID(ogc.ID),
Jobid: database.MustToDBUUID(ogc.JobID),
Mincleanversion: ogc.MinCleanVersion,
Mintextversion: ogc.MinTextVersion,
Fields: []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"])),
})
assert.Nil(t, err)
assert.EqualExportedValues(t, ogc, *c)
}
internal/job/collector/service.go
+12
View File
@@ -2,8 +2,20 @@ package collector
import (
"queryorchestration/internal/database"
"github.com/google/uuid"
)
type Collector struct {
ID uuid.UUID
JobID uuid.UUID
MinCleanVersion int32
MinTextVersion int32
ActiveVersion int32
LatestVersion int32
Fields map[string]uuid.UUID
}
type Service struct {
db *database.Connection
}
internal/job/collector/service_test.go
-41
View File
@@ -1,13 +1,11 @@
package collector_test
import (
"context"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/job/collector"
"testing"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
)
@@ -26,42 +24,3 @@ func TestService(t *testing.T) {
svc := collector.New(db)
assert.NotNil(t, svc)
}
func TestByJobId(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
jobID := uuid.New()
fullCollector := collector.Collector{
ID: uuid.New(),
MinCleanVersion: int32(1),
MinTextVersion: int32(1),
}
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(database.MustToDBUUID(jobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(database.MustToDBUUID(fullCollector.ID), database.MustToDBUUID(jobID), fullCollector.MinCleanVersion, fullCollector.MinTextVersion),
)
coll, err := collector.NewByJobId(ctx, db, jobID)
assert.Nil(t, err)
assert.EqualExportedValues(t, fullCollector, *coll)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(database.MustToDBUUID(jobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}),
)
_, err = collector.NewByJobId(ctx, db, jobID)
assert.EqualError(t, err, "no rows in result set")
}
internal/query/document/service.go
+19 -6
View File
@@ -19,28 +19,41 @@ type Document struct {
TextVersion int32 `json:"textVersion" validate:"required,gt=0"`
}
type Service struct {
db *database.Connection
type Services struct {
Collector *collector.Service
}
func New(db *database.Connection) *Service {
type Service struct {
db *database.Connection
svc *Services
}
func New(db *database.Connection, svc *Services) *Service {
return &Service{
db,
svc,
}
}
func (s *Service) Sync(ctx context.Context, doc *Document) error {
collector, err := collector.NewByJobId(ctx, s.db, doc.JobID)
coll, err := s.svc.Collector.GetByJobID(ctx, doc.JobID)
if err != nil {
return err
}
results, err := s.getResults(ctx, doc.ID, collector)
results, err := s.getResults(ctx, doc.ID, coll)
if err != nil {
return err
}
queue, err := queryqueue.New(ctx, s.db, collector, results, doc.ID, doc.CleanVersion, doc.TextVersion)
queue, err := queryqueue.New(ctx, &queryqueue.NewConfig{
DB: s.db,
Collector: coll,
Results: results,
DocumentID: doc.ID,
CleanVersion: doc.CleanVersion,
TextVersion: doc.TextVersion,
})
if err != nil {
return err
}
internal/query/document/sync_test.go
+53 -38
View File
@@ -5,6 +5,7 @@ import (
"errors"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/job/collector"
"queryorchestration/internal/query/document"
"testing"
@@ -33,29 +34,33 @@ func TestSyncIsSynced(t *testing.T) {
Name: "document_name",
}
dbCollectorId := database.MustToDBUUID(uuid.New())
dbJobID := database.MustToDBUUID(doc.JobID)
minCleanVersion := int32(1)
minTextVersion := int32(1)
qV := int32(1)
queryVersion := int32(1)
coll := collector.Collector{
ID: uuid.New(),
JobID: doc.JobID,
MinCleanVersion: int32(1),
MinTextVersion: int32(2),
}
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion),
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(coll.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")),
)
pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), coll.MinCleanVersion, coll.MinTextVersion).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}).
AddRow(pgtype.UUID{}, pgtype.UUID{}, qV),
AddRow(pgtype.UUID{}, pgtype.UUID{}, queryVersion),
)
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorId).
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(database.MustToDBUUID(coll.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
AddRow(dbCollectorId, pgtype.UUID{}, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, &qV, []pgtype.UUID{}),
AddRow(database.MustToDBUUID(coll.ID), pgtype.UUID{}, repository.QuerytypeJsonExtractor, queryVersion, []pgtype.UUID{}),
)
docSvc := document.New(db)
docSvc := document.New(db, &document.Services{
Collector: collector.New(db),
})
err = docSvc.Sync(ctx, &doc)
assert.Nil(t, err)
}
@@ -85,49 +90,55 @@ func TestSyncDBFail(t *testing.T) {
minCleanVersion := int32(1)
minTextVersion := int32(1)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}),
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}),
)
docSvc := document.New(db)
docSvc := document.New(db, &document.Services{
Collector: collector.New(db),
})
err = docSvc.Sync(ctx, &doc)
assert.EqualError(t, err, "no rows in result set")
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion),
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion, int32(1), int32(2), []byte("")),
)
dbErr := "database failure"
pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
WillReturnError(errors.New(dbErr))
docSvc = document.New(db)
docSvc = document.New(db, &document.Services{
Collector: collector.New(db),
})
err = docSvc.Sync(ctx, &doc)
assert.EqualError(t, err, dbErr)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion),
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion, int32(1), int32(2), []byte("")),
)
pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}),
)
dbErr = "database failure"
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorId).
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorId).
WillReturnError(errors.New(dbErr))
docSvc = document.New(db)
docSvc = document.New(db, &document.Services{
Collector: collector.New(db),
})
err = docSvc.Sync(ctx, &doc)
assert.EqualError(t, err, dbErr)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion),
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion, int32(1), int32(2), []byte("")),
)
qV := int32(1)
reqID := database.MustToDBUUID(uuid.New())
@@ -138,16 +149,18 @@ func TestSyncDBFail(t *testing.T) {
AddRow(resID, reqID, qV),
)
dbErr = "database failure"
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorId).
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorId).
WillReturnRows(
pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
AddRow(dbCollectorId, dbQueryID, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, &qV, []pgtype.UUID{reqID}).
AddRow(dbCollectorId, reqID, repository.NullQuerytype{Querytype: repository.QuerytypeContextFull, Valid: true}, &qV, []pgtype.UUID{}),
AddRow(dbCollectorId, dbQueryID, repository.QuerytypeJsonExtractor, qV, []pgtype.UUID{reqID}).
AddRow(dbCollectorId, reqID, repository.QuerytypeContextFull, qV, []pgtype.UUID{}),
)
pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{resID}).
WillReturnError(errors.New(dbErr))
docSvc = document.New(db)
docSvc = document.New(db, &document.Services{
Collector: collector.New(db),
})
err = docSvc.Sync(ctx, &doc)
assert.EqualError(t, err, dbErr)
}
@@ -177,27 +190,29 @@ func TestSync(t *testing.T) {
minCleanVersion := int32(1)
minTextVersion := int32(1)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion),
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion, int32(1), int32(2), []byte("")),
)
pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}),
)
qV := int32(1)
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorId).
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorId).
WillReturnRows(
pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
AddRow(dbCollectorId, dbQueryID, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, &qV, []pgtype.UUID{}),
AddRow(dbCollectorId, dbQueryID, repository.QuerytypeJsonExtractor, qV, []pgtype.UUID{}),
)
pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{}).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "value"}),
)
docSvc := document.New(db)
docSvc := document.New(db, &document.Services{
Collector: collector.New(db),
})
err = docSvc.Sync(ctx, &doc)
assert.EqualError(t, err, "JSON Extraction requires 1 result")
}
internal/query/processor/parse.go
+3 -3
View File
@@ -65,21 +65,21 @@ func ToDBNullQueryType(t Type) (repository.NullQuerytype, error) {
return repository.NullQuerytype{Querytype: dbType, Valid: true}, nil
}
func ParseDBCollectorQuery(q *repository.GetCollectorQueriesRow) (*Query, error) {
func ParseDBCollectorQuery(q *repository.Collectorquerydependencytree) (*Query, error) {
var reqQueryIDs *[]uuid.UUID
if len(q.Requiredids) > 0 {
ids := database.MustToUUIDArray(q.Requiredids)
reqQueryIDs = &ids
}
qType, err := ParseDBNullType(q.Type)
qType, err := ParseDBType(q.Type)
if err != nil {
return nil, err
}
return &Query{
ID: database.MustToUUID(q.Queryid),
Version: *q.Queryversion,
Version: q.Queryversion,
Type: qType,
RequiredQueryIDs: reqQueryIDs,
}, nil
internal/query/processor/parse_test.go
+5 -6
View File
@@ -11,22 +11,21 @@ import (
)
func TestParseDBCollectorQuery(t *testing.T) {
qV := int32(0)
dbResult := repository.GetCollectorQueriesRow{
dbResult := repository.Collectorquerydependencytree{
Collectorid: pgtype.UUID{},
Queryid: pgtype.UUID{},
Requiredids: []pgtype.UUID{},
Type: repository.NullQuerytype{Valid: true, Querytype: repository.QuerytypeJsonExtractor},
Queryversion: &qV,
Type: repository.QuerytypeJsonExtractor,
Queryversion: 1,
}
value, err := queryprocessor.ParseDBCollectorQuery(&dbResult)
assert.Nil(t, err)
assert.Equal(t, uuid.Nil, value.ID)
assert.Nil(t, value.RequiredQueryIDs)
assert.Equal(t, int32(0), value.Version)
assert.Equal(t, int32(1), value.Version)
assert.Equal(t, queryprocessor.Type(queryprocessor.TypeJsonExtractor), value.Type)
dbResult.Type = repository.NullQuerytype{}
dbResult.Type = repository.Querytype("")
_, err = queryprocessor.ParseDBCollectorQuery(&dbResult)
assert.EqualError(t, err, "invalid database query type")
}
internal/query/queue/create.go
+1 -1
View File
@@ -33,7 +33,7 @@ func (c *Queue) getCollectorQueries(ctx context.Context) error {
id := database.MustToDBUUID(c.collector.ID)
queries, err := c.db.Queries.GetCollectorQueries(ctx, id)
queries, err := c.db.Queries.ListCollectorQueries(ctx, id)
if err != nil {
return err
}
internal/query/queue/create_test.go
+3 -3
View File
@@ -71,13 +71,13 @@ func TestGetCollectorQueries(t *testing.T) {
if q.RequiredQueryIDs != nil {
dbReqIDs = database.MustToDBUUIDArray(*q.RequiredQueryIDs)
}
ty, err := queryprocessor.ToDBNullQueryType(q.Type)
ty, err := queryprocessor.ToDBQueryType(q.Type)
assert.Nil(t, err)
rows = rows.
AddRow(dbCollectorID, dbID, ty, &q.Version, dbReqIDs)
AddRow(dbCollectorID, dbID, ty, q.Version, dbReqIDs)
}
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
err = svc.getCollectorQueries(ctx)
assert.Nil(t, err)
internal/query/queue/execute_test.go
+10 -17
View File
@@ -29,19 +29,10 @@ func TestExecute(t *testing.T) {
Queries: queries,
Pool: pool,
}
jobID := uuid.New()
dbJobID := database.MustToDBUUID(jobID)
collectorID := uuid.New()
dbCollectorID := database.MustToDBUUID(collectorID)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorID, dbJobID, int32(1), int32(1)),
)
coll, err := collector.NewByJobId(ctx, db, jobID)
assert.Nil(t, err)
coll := collector.Collector{
ID: uuid.New(),
}
dbCollectorID := database.MustToDBUUID(coll.ID)
queryOneID := uuid.New()
queryOneVersion := int32(1)
@@ -74,13 +65,13 @@ func TestExecute(t *testing.T) {
if q.RequiredQueryIDs != nil {
dbReqIDs = database.MustToDBUUIDArray(*q.RequiredQueryIDs)
}
ty, err := queryprocessor.ToDBNullQueryType(q.Type)
ty, err := queryprocessor.ToDBQueryType(q.Type)
assert.Nil(t, err)
rows = rows.
AddRow(dbCollectorID, dbID, ty, &q.Version, dbReqIDs)
AddRow(dbCollectorID, dbID, ty, q.Version, dbReqIDs)
}
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
contextResultID := uuid.New()
results := []*result.Result{
@@ -102,7 +93,9 @@ func TestExecute(t *testing.T) {
cleanVersion := int32(1)
textVersion := int32(1)
q, err := queryqueue.New(ctx, db, coll, results, docID, cleanVersion, textVersion)
q, err := queryqueue.New(ctx, &queryqueue.NewConfig{
db, &coll, results, docID, cleanVersion, textVersion,
})
assert.Nil(t, err)
assert.Equal(t, expectedQueries, q.GetQueue())
internal/query/queue/service.go
+17 -8
View File
@@ -16,19 +16,28 @@ type Queue struct {
results []*result.Result
collector *collector.Collector
db *database.Connection
documentId uuid.UUID
cleanVersion int32
textVersion int32
documentId uuid.UUID
}
func New(ctx context.Context, db *database.Connection, coll *collector.Collector, results []*result.Result, docId uuid.UUID, cleanVersion int32, textVersion int32) (*Queue, error) {
type NewConfig struct {
DB *database.Connection
Collector *collector.Collector
Results []*result.Result
DocumentID uuid.UUID
CleanVersion int32
TextVersion int32
}
func New(ctx context.Context, cfg *NewConfig) (*Queue, error) {
queue := Queue{
db: db,
results: results,
collector: coll,
documentId: docId,
cleanVersion: cleanVersion,
textVersion: textVersion,
db: cfg.DB,
results: cfg.Results,
collector: cfg.Collector,
documentId: cfg.DocumentID,
cleanVersion: cfg.CleanVersion,
textVersion: cfg.TextVersion,
}
err := queue.getCollectorQueries(ctx)
internal/query/queue/service_test.go
+18 -31
View File
@@ -29,19 +29,11 @@ func TestService(t *testing.T) {
Queries: queries,
Pool: pool,
}
jobID := uuid.New()
dbJobID := database.MustToDBUUID(jobID)
collectorID := uuid.New()
dbCollectorID := database.MustToDBUUID(collectorID)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorID, dbJobID, int32(1), int32(1)),
)
coll, err := collector.NewByJobId(ctx, db, jobID)
assert.Nil(t, err)
coll := collector.Collector{
ID: uuid.New(),
}
dbCollectorID := database.MustToDBUUID(coll.ID)
queryOneID := uuid.New()
queryOneVersion := int32(1)
@@ -74,13 +66,13 @@ func TestService(t *testing.T) {
if q.RequiredQueryIDs != nil {
dbReqIDs = database.MustToDBUUIDArray(*q.RequiredQueryIDs)
}
ty, err := queryprocessor.ToDBNullQueryType(q.Type)
ty, err := queryprocessor.ToDBQueryType(q.Type)
assert.Nil(t, err)
rows = rows.
AddRow(dbCollectorID, dbID, ty, &q.Version, dbReqIDs)
AddRow(dbCollectorID, dbID, ty, q.Version, dbReqIDs)
}
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
contextResultID := uuid.New()
results := []*result.Result{
@@ -102,7 +94,9 @@ func TestService(t *testing.T) {
cleanVersion := int32(1)
textVersion := int32(1)
q, err := queryqueue.New(ctx, db, coll, results, docID, cleanVersion, textVersion)
q, err := queryqueue.New(ctx, &queryqueue.NewConfig{
db, &coll, results, docID, cleanVersion, textVersion,
})
assert.Nil(t, err)
assert.Equal(t, expectedQueries, q.GetQueue())
}
@@ -119,22 +113,13 @@ func TestQueueFail(t *testing.T) {
Queries: queries,
Pool: pool,
}
jobID := uuid.New()
dbJobID := database.MustToDBUUID(jobID)
collectorID := uuid.New()
dbCollectorID := database.MustToDBUUID(collectorID)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorID, dbJobID, int32(1), int32(1)),
)
coll, err := collector.NewByJobId(ctx, db, jobID)
assert.Nil(t, err)
coll := collector.Collector{
ID: uuid.New(),
}
dbCollectorID := database.MustToDBUUID(coll.ID)
dbErr := "database failure"
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorID).
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorID).
WillReturnError(errors.New(dbErr))
results := []*result.Result{}
@@ -143,6 +128,8 @@ func TestQueueFail(t *testing.T) {
cleanVersion := int32(1)
textVersion := int32(1)
_, err = queryqueue.New(ctx, db, coll, results, docID, cleanVersion, textVersion)
_, err = queryqueue.New(ctx, &queryqueue.NewConfig{
db, &coll, results, docID, cleanVersion, textVersion,
})
assert.EqualError(t, err, dbErr)
}
pkg/queryService/api.gen.go
+88 -203
View File
@@ -97,20 +97,47 @@ type IdMessage struct {
// JobCollector JobCollector model.
type JobCollector struct {
// ExampleProperty Example property for JobCollector.
ExampleProperty *string `json:"example_property,omitempty"`
// ActiveVersion The active version of the collector.
ActiveVersion int32 `json:"active_version"`
// Fields The fields in the job collector.
Fields []JobCollectorField `json:"fields"`
// JobId The ID of the associated job.
JobId string `json:"job_id"`
// LatestVersion The latest version of the collector.
LatestVersion int32 `json:"latest_version"`
// MinimumCleanerVersion The minimum version for the document cleaner.
MinimumCleanerVersion int32 `json:"minimum_cleaner_version"`
// MinimumTextVersion The minimum version for the text parser.
MinimumTextVersion int32 `json:"minimum_text_version"`
}
// JobCollectorCreate Payload for creating a JobCollector.
type JobCollectorCreate struct {
// ExampleProperty Example property for JobCollectorCreate.
ExampleProperty *string `json:"example_property,omitempty"`
// JobCollectorField The field properties for the job collector.
type JobCollectorField struct {
// Name The output field name.
Name string `json:"name"`
// QueryId The query id that will populate the result.
QueryId string `json:"query_id"`
}
// JobCollectorUpdate Payload for updating a JobCollector.
type JobCollectorUpdate struct {
// ExampleProperty Example property for JobCollectorUpdate.
ExampleProperty *string `json:"example_property,omitempty"`
// ActiveVersion The active version of the collector.
ActiveVersion *int32 `json:"active_version,omitempty"`
// Fields The fields in the job collector.
Fields *[]JobCollectorField `json:"fields,omitempty"`
// MinimumCleanerVersion The minimum version for the document cleaner.
MinimumCleanerVersion *int32 `json:"minimum_cleaner_version,omitempty"`
// MinimumTextVersion The minimum version for the text parser.
MinimumTextVersion *int32 `json:"minimum_text_version,omitempty"`
}
// ListQueries defines model for ListQueries.
@@ -185,11 +212,8 @@ type QueryUpdate struct {
// TriggerExportJSONRequestBody defines body for TriggerExport for application/json ContentType.
type TriggerExportJSONRequestBody = ExportTrigger
// CreateJobCollectorJSONRequestBody defines body for CreateJobCollector for application/json ContentType.
type CreateJobCollectorJSONRequestBody = JobCollectorCreate
// UpdateJobCollectorJSONRequestBody defines body for UpdateJobCollector for application/json ContentType.
type UpdateJobCollectorJSONRequestBody = JobCollectorUpdate
// UpdateJobCollectorByJobIdJSONRequestBody defines body for UpdateJobCollectorByJobId for application/json ContentType.
type UpdateJobCollectorByJobIdJSONRequestBody = JobCollectorUpdate
// CreateQueryJSONRequestBody defines body for CreateQuery for application/json ContentType.
type CreateQueryJSONRequestBody = QueryCreate
@@ -281,18 +305,13 @@ type ClientInterface interface {
// ExportState request
ExportState(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error)
// CreateJobCollectorWithBody request with any body
CreateJobCollectorWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error)
// GetJobCollectorByJobId request
GetJobCollectorByJobId(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error)
CreateJobCollector(ctx context.Context, body CreateJobCollectorJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error)
// UpdateJobCollectorByJobIdWithBody request with any body
UpdateJobCollectorByJobIdWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error)
// GetJobCollectorById request
GetJobCollectorById(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error)
// UpdateJobCollectorWithBody request with any body
UpdateJobCollectorWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error)
UpdateJobCollector(ctx context.Context, id string, body UpdateJobCollectorJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error)
UpdateJobCollectorByJobId(ctx context.Context, id string, body UpdateJobCollectorByJobIdJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error)
// ListQueries request
ListQueries(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error)
@@ -355,8 +374,8 @@ func (c *Client) ExportState(ctx context.Context, id string, reqEditors ...Reque
return c.Client.Do(req)
}
func (c *Client) CreateJobCollectorWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) {
req, err := NewCreateJobCollectorRequestWithBody(c.Server, contentType, body)
func (c *Client) GetJobCollectorByJobId(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) {
req, err := NewGetJobCollectorByJobIdRequest(c.Server, id)
if err != nil {
return nil, err
}
@@ -367,8 +386,8 @@ func (c *Client) CreateJobCollectorWithBody(ctx context.Context, contentType str
return c.Client.Do(req)
}
func (c *Client) CreateJobCollector(ctx context.Context, body CreateJobCollectorJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) {
req, err := NewCreateJobCollectorRequest(c.Server, body)
func (c *Client) UpdateJobCollectorByJobIdWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) {
req, err := NewUpdateJobCollectorByJobIdRequestWithBody(c.Server, id, contentType, body)
if err != nil {
return nil, err
}
@@ -379,32 +398,8 @@ func (c *Client) CreateJobCollector(ctx context.Context, body CreateJobCollector
return c.Client.Do(req)
}
func (c *Client) GetJobCollectorById(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) {
req, err := NewGetJobCollectorByIdRequest(c.Server, id)
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
if err := c.applyEditors(ctx, req, reqEditors); err != nil {
return nil, err
}
return c.Client.Do(req)
}
func (c *Client) UpdateJobCollectorWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) {
req, err := NewUpdateJobCollectorRequestWithBody(c.Server, id, contentType, body)
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
if err := c.applyEditors(ctx, req, reqEditors); err != nil {
return nil, err
}
return c.Client.Do(req)
}
func (c *Client) UpdateJobCollector(ctx context.Context, id string, body UpdateJobCollectorJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) {
req, err := NewUpdateJobCollectorRequest(c.Server, id, body)
func (c *Client) UpdateJobCollectorByJobId(ctx context.Context, id string, body UpdateJobCollectorByJobIdJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) {
req, err := NewUpdateJobCollectorByJobIdRequest(c.Server, id, body)
if err != nil {
return nil, err
}
@@ -543,7 +538,7 @@ func NewTriggerExportRequestWithBody(server string, contentType string, body io.
return nil, err
}
operationPath := fmt.Sprintf("/export")
operationPath := fmt.Sprintf("/job/export")
if operationPath[0] == '/' {
operationPath = "." + operationPath
}
@@ -579,7 +574,7 @@ func NewExportStateRequest(server string, id string) (*http.Request, error) {
return nil, err
}
operationPath := fmt.Sprintf("/export/%s", pathParam0)
operationPath := fmt.Sprintf("/job/export/%s", pathParam0)
if operationPath[0] == '/' {
operationPath = "." + operationPath
}
@@ -597,48 +592,8 @@ func NewExportStateRequest(server string, id string) (*http.Request, error) {
return req, nil
}
// NewCreateJobCollectorRequest calls the generic CreateJobCollector builder with application/json body
func NewCreateJobCollectorRequest(server string, body CreateJobCollectorJSONRequestBody) (*http.Request, error) {
var bodyReader io.Reader
buf, err := json.Marshal(body)
if err != nil {
return nil, err
}
bodyReader = bytes.NewReader(buf)
return NewCreateJobCollectorRequestWithBody(server, "application/json", bodyReader)
}
// NewCreateJobCollectorRequestWithBody generates requests for CreateJobCollector with any type of body
func NewCreateJobCollectorRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) {
var err error
serverURL, err := url.Parse(server)
if err != nil {
return nil, err
}
operationPath := fmt.Sprintf("/job-collectors")
if operationPath[0] == '/' {
operationPath = "." + operationPath
}
queryURL, err := serverURL.Parse(operationPath)
if err != nil {
return nil, err
}
req, err := http.NewRequest("POST", queryURL.String(), body)
if err != nil {
return nil, err
}
req.Header.Add("Content-Type", contentType)
return req, nil
}
// NewGetJobCollectorByIdRequest generates requests for GetJobCollectorById
func NewGetJobCollectorByIdRequest(server string, id string) (*http.Request, error) {
// NewGetJobCollectorByJobIdRequest generates requests for GetJobCollectorByJobId
func NewGetJobCollectorByJobIdRequest(server string, id string) (*http.Request, error) {
var err error
var pathParam0 string
@@ -653,7 +608,7 @@ func NewGetJobCollectorByIdRequest(server string, id string) (*http.Request, err
return nil, err
}
operationPath := fmt.Sprintf("/job-collectors/%s", pathParam0)
operationPath := fmt.Sprintf("/job/%s/collector", pathParam0)
if operationPath[0] == '/' {
operationPath = "." + operationPath
}
@@ -671,19 +626,19 @@ func NewGetJobCollectorByIdRequest(server string, id string) (*http.Request, err
return req, nil
}
// NewUpdateJobCollectorRequest calls the generic UpdateJobCollector builder with application/json body
func NewUpdateJobCollectorRequest(server string, id string, body UpdateJobCollectorJSONRequestBody) (*http.Request, error) {
// NewUpdateJobCollectorByJobIdRequest calls the generic UpdateJobCollectorByJobId builder with application/json body
func NewUpdateJobCollectorByJobIdRequest(server string, id string, body UpdateJobCollectorByJobIdJSONRequestBody) (*http.Request, error) {
var bodyReader io.Reader
buf, err := json.Marshal(body)
if err != nil {
return nil, err
}
bodyReader = bytes.NewReader(buf)
return NewUpdateJobCollectorRequestWithBody(server, id, "application/json", bodyReader)
return NewUpdateJobCollectorByJobIdRequestWithBody(server, id, "application/json", bodyReader)
}
// NewUpdateJobCollectorRequestWithBody generates requests for UpdateJobCollector with any type of body
func NewUpdateJobCollectorRequestWithBody(server string, id string, contentType string, body io.Reader) (*http.Request, error) {
// NewUpdateJobCollectorByJobIdRequestWithBody generates requests for UpdateJobCollectorByJobId with any type of body
func NewUpdateJobCollectorByJobIdRequestWithBody(server string, id string, contentType string, body io.Reader) (*http.Request, error) {
var err error
var pathParam0 string
@@ -698,7 +653,7 @@ func NewUpdateJobCollectorRequestWithBody(server string, id string, contentType
return nil, err
}
operationPath := fmt.Sprintf("/job-collectors/%s", pathParam0)
operationPath := fmt.Sprintf("/job/%s/collector", pathParam0)
if operationPath[0] == '/' {
operationPath = "." + operationPath
}
@@ -708,7 +663,7 @@ func NewUpdateJobCollectorRequestWithBody(server string, id string, contentType
return nil, err
}
req, err := http.NewRequest("PUT", queryURL.String(), body)
req, err := http.NewRequest("PATCH", queryURL.String(), body)
if err != nil {
return nil, err
}
@@ -998,18 +953,13 @@ type ClientWithResponsesInterface interface {
// ExportStateWithResponse request
ExportStateWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*ExportStateResponse, error)
// CreateJobCollectorWithBodyWithResponse request with any body
CreateJobCollectorWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateJobCollectorResponse, error)
// GetJobCollectorByJobIdWithResponse request
GetJobCollectorByJobIdWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetJobCollectorByJobIdResponse, error)
CreateJobCollectorWithResponse(ctx context.Context, body CreateJobCollectorJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateJobCollectorResponse, error)
// UpdateJobCollectorByJobIdWithBodyWithResponse request with any body
UpdateJobCollectorByJobIdWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UpdateJobCollectorByJobIdResponse, error)
// GetJobCollectorByIdWithResponse request
GetJobCollectorByIdWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetJobCollectorByIdResponse, error)
// UpdateJobCollectorWithBodyWithResponse request with any body
UpdateJobCollectorWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UpdateJobCollectorResponse, error)
UpdateJobCollectorWithResponse(ctx context.Context, id string, body UpdateJobCollectorJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateJobCollectorResponse, error)
UpdateJobCollectorByJobIdWithResponse(ctx context.Context, id string, body UpdateJobCollectorByJobIdJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateJobCollectorByJobIdResponse, error)
// ListQueriesWithResponse request
ListQueriesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*ListQueriesResponse, error)
@@ -1080,36 +1030,14 @@ func (r ExportStateResponse) StatusCode() int {
return 0
}
type CreateJobCollectorResponse struct {
Body []byte
HTTPResponse *http.Response
JSON201 *IdMessage
}
// Status returns HTTPResponse.Status
func (r CreateJobCollectorResponse) Status() string {
if r.HTTPResponse != nil {
return r.HTTPResponse.Status
}
return http.StatusText(0)
}
// StatusCode returns HTTPResponse.StatusCode
func (r CreateJobCollectorResponse) StatusCode() int {
if r.HTTPResponse != nil {
return r.HTTPResponse.StatusCode
}
return 0
}
type GetJobCollectorByIdResponse struct {
type GetJobCollectorByJobIdResponse struct {
Body []byte
HTTPResponse *http.Response
JSON200 *JobCollector
}
// Status returns HTTPResponse.Status
func (r GetJobCollectorByIdResponse) Status() string {
func (r GetJobCollectorByJobIdResponse) Status() string {
if r.HTTPResponse != nil {
return r.HTTPResponse.Status
}
@@ -1117,20 +1045,20 @@ func (r GetJobCollectorByIdResponse) Status() string {
}
// StatusCode returns HTTPResponse.StatusCode
func (r GetJobCollectorByIdResponse) StatusCode() int {
func (r GetJobCollectorByJobIdResponse) StatusCode() int {
if r.HTTPResponse != nil {
return r.HTTPResponse.StatusCode
}
return 0
}
type UpdateJobCollectorResponse struct {
type UpdateJobCollectorByJobIdResponse struct {
Body []byte
HTTPResponse *http.Response
}
// Status returns HTTPResponse.Status
func (r UpdateJobCollectorResponse) Status() string {
func (r UpdateJobCollectorByJobIdResponse) Status() string {
if r.HTTPResponse != nil {
return r.HTTPResponse.Status
}
@@ -1138,7 +1066,7 @@ func (r UpdateJobCollectorResponse) Status() string {
}
// StatusCode returns HTTPResponse.StatusCode
func (r UpdateJobCollectorResponse) StatusCode() int {
func (r UpdateJobCollectorByJobIdResponse) StatusCode() int {
if r.HTTPResponse != nil {
return r.HTTPResponse.StatusCode
}
@@ -1301,47 +1229,30 @@ func (c *ClientWithResponses) ExportStateWithResponse(ctx context.Context, id st
return ParseExportStateResponse(rsp)
}
// CreateJobCollectorWithBodyWithResponse request with arbitrary body returning *CreateJobCollectorResponse
func (c *ClientWithResponses) CreateJobCollectorWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateJobCollectorResponse, error) {
rsp, err := c.CreateJobCollectorWithBody(ctx, contentType, body, reqEditors...)
// GetJobCollectorByJobIdWithResponse request returning *GetJobCollectorByJobIdResponse
func (c *ClientWithResponses) GetJobCollectorByJobIdWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetJobCollectorByJobIdResponse, error) {
rsp, err := c.GetJobCollectorByJobId(ctx, id, reqEditors...)
if err != nil {
return nil, err
}
return ParseCreateJobCollectorResponse(rsp)
return ParseGetJobCollectorByJobIdResponse(rsp)
}
func (c *ClientWithResponses) CreateJobCollectorWithResponse(ctx context.Context, body CreateJobCollectorJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateJobCollectorResponse, error) {
rsp, err := c.CreateJobCollector(ctx, body, reqEditors...)
// UpdateJobCollectorByJobIdWithBodyWithResponse request with arbitrary body returning *UpdateJobCollectorByJobIdResponse
func (c *ClientWithResponses) UpdateJobCollectorByJobIdWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UpdateJobCollectorByJobIdResponse, error) {
rsp, err := c.UpdateJobCollectorByJobIdWithBody(ctx, id, contentType, body, reqEditors...)
if err != nil {
return nil, err
}
return ParseCreateJobCollectorResponse(rsp)
return ParseUpdateJobCollectorByJobIdResponse(rsp)
}
// GetJobCollectorByIdWithResponse request returning *GetJobCollectorByIdResponse
func (c *ClientWithResponses) GetJobCollectorByIdWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetJobCollectorByIdResponse, error) {
rsp, err := c.GetJobCollectorById(ctx, id, reqEditors...)
func (c *ClientWithResponses) UpdateJobCollectorByJobIdWithResponse(ctx context.Context, id string, body UpdateJobCollectorByJobIdJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateJobCollectorByJobIdResponse, error) {
rsp, err := c.UpdateJobCollectorByJobId(ctx, id, body, reqEditors...)
if err != nil {
return nil, err
}
return ParseGetJobCollectorByIdResponse(rsp)
}
// UpdateJobCollectorWithBodyWithResponse request with arbitrary body returning *UpdateJobCollectorResponse
func (c *ClientWithResponses) UpdateJobCollectorWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UpdateJobCollectorResponse, error) {
rsp, err := c.UpdateJobCollectorWithBody(ctx, id, contentType, body, reqEditors...)
if err != nil {
return nil, err
}
return ParseUpdateJobCollectorResponse(rsp)
}
func (c *ClientWithResponses) UpdateJobCollectorWithResponse(ctx context.Context, id string, body UpdateJobCollectorJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateJobCollectorResponse, error) {
rsp, err := c.UpdateJobCollector(ctx, id, body, reqEditors...)
if err != nil {
return nil, err
}
return ParseUpdateJobCollectorResponse(rsp)
return ParseUpdateJobCollectorByJobIdResponse(rsp)
}
// ListQueriesWithResponse request returning *ListQueriesResponse
@@ -1474,41 +1385,15 @@ func ParseExportStateResponse(rsp *http.Response) (*ExportStateResponse, error)
return response, nil
}
// ParseCreateJobCollectorResponse parses an HTTP response from a CreateJobCollectorWithResponse call
func ParseCreateJobCollectorResponse(rsp *http.Response) (*CreateJobCollectorResponse, error) {
// ParseGetJobCollectorByJobIdResponse parses an HTTP response from a GetJobCollectorByJobIdWithResponse call
func ParseGetJobCollectorByJobIdResponse(rsp *http.Response) (*GetJobCollectorByJobIdResponse, error) {
bodyBytes, err := io.ReadAll(rsp.Body)
defer func() { _ = rsp.Body.Close() }()
if err != nil {
return nil, err
}
response := &CreateJobCollectorResponse{
Body: bodyBytes,
HTTPResponse: rsp,
}
switch {
case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 201:
var dest IdMessage
if err := json.Unmarshal(bodyBytes, &dest); err != nil {
return nil, err
}
response.JSON201 = &dest
}
return response, nil
}
// ParseGetJobCollectorByIdResponse parses an HTTP response from a GetJobCollectorByIdWithResponse call
func ParseGetJobCollectorByIdResponse(rsp *http.Response) (*GetJobCollectorByIdResponse, error) {
bodyBytes, err := io.ReadAll(rsp.Body)
defer func() { _ = rsp.Body.Close() }()
if err != nil {
return nil, err
}
response := &GetJobCollectorByIdResponse{
response := &GetJobCollectorByJobIdResponse{
Body: bodyBytes,
HTTPResponse: rsp,
}
@@ -1526,15 +1411,15 @@ func ParseGetJobCollectorByIdResponse(rsp *http.Response) (*GetJobCollectorByIdR
return response, nil
}
// ParseUpdateJobCollectorResponse parses an HTTP response from a UpdateJobCollectorWithResponse call
func ParseUpdateJobCollectorResponse(rsp *http.Response) (*UpdateJobCollectorResponse, error) {
// ParseUpdateJobCollectorByJobIdResponse parses an HTTP response from a UpdateJobCollectorByJobIdWithResponse call
func ParseUpdateJobCollectorByJobIdResponse(rsp *http.Response) (*UpdateJobCollectorByJobIdResponse, error) {
bodyBytes, err := io.ReadAll(rsp.Body)
defer func() { _ = rsp.Body.Close() }()
if err != nil {
return nil, err
}
response := &UpdateJobCollectorResponse{
response := &UpdateJobCollectorByJobIdResponse{
Body: bodyBytes,
HTTPResponse: rsp,
}
serviceAPIs/queryService.yaml
+69 -43
View File
@@ -156,32 +156,9 @@ paths:
'400':
description: Invalid request body.
/job-collectors:
post:
operationId: createJobCollector
tags:
- JobCollectorService
summary: Create a job collector
description: Creates a new job collector.
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/JobCollectorCreate'
responses:
'201':
description: Job collector created successfully.
content:
application/json:
schema:
$ref: '#/components/schemas/IdMessage'
'400':
description: Invalid request body.
/job-collectors/{id}:
/job/{id}/collector:
get:
operationId: getJobCollectorById
operationId: getJobCollectorByJobId
tags:
- JobCollectorService
summary: Get a job collector by ID
@@ -192,7 +169,7 @@ paths:
required: true
schema:
type: string
description: The ID of the job collector to retrieve.
description: The job ID for the collector.
responses:
'200':
description: Job collector details.
@@ -202,8 +179,8 @@ paths:
$ref: '#/components/schemas/JobCollector'
'404':
description: Job collector not found.
put:
operationId: updateJobCollector
patch:
operationId: updateJobCollectorByJobId
tags:
- JobCollectorService
summary: Update a job collector
@@ -229,7 +206,7 @@ paths:
'404':
description: Job collector not found.
/export:
/job/export:
post:
operationId: triggerExport
tags:
@@ -254,7 +231,7 @@ paths:
'404':
description: Job not found.
/export/{id}:
/job/export/{id}:
get:
operationId: exportState
tags:
@@ -398,27 +375,76 @@ components:
JobCollector:
type: object
properties:
example_property:
job_id:
type: string
description: Example property for JobCollector.
description: The ID of the associated job.
active_version:
type: integer
format: int32
description: The active version of the collector.
latest_version:
type: integer
format: int32
description: The latest version of the collector.
minimum_cleaner_version:
type: integer
format: int32
description: The minimum version for the document cleaner.
minimum_text_version:
type: integer
format: int32
description: The minimum version for the text parser.
fields:
type: array
description: The fields in the job collector.
items:
$ref: '#/components/schemas/JobCollectorField'
description: JobCollector model.
JobCollectorCreate:
type: object
properties:
example_property:
type: string
description: Example property for JobCollectorCreate.
description: Payload for creating a JobCollector.
required:
- id
- job_id
- fields
- minimum_cleaner_version
- minimum_text_version
- latest_version
- active_version
JobCollectorUpdate:
type: object
properties:
example_property:
type: string
description: Example property for JobCollectorUpdate.
minimum_cleaner_version:
type: integer
format: int32
description: The minimum version for the document cleaner.
minimum_text_version:
type: integer
format: int32
description: The minimum version for the text parser.
active_version:
type: integer
format: int32
description: The active version of the collector.
fields:
type: array
description: The fields in the job collector.
items:
$ref: '#/components/schemas/JobCollectorField'
description: Payload for updating a JobCollector.
JobCollectorField:
type: object
description: The field properties for the job collector.
properties:
name:
type: string
description: The output field name.
query_id:
type: string
description: The query id that will populate the result.
required:
- name
- query_id
ExportTrigger:
type: object
properties:
sqlc.yml
+1 -1
View File
@@ -13,7 +13,7 @@ sql:
- no-delete
- no-pg
- no-seq-scan
- too-costly
# - too-costly
database:
managed: true
gen:
test/queryService/jobcollectorservice_test.go
+41 -7
View File
@@ -11,6 +11,7 @@ import (
)
func TestJobCollectorService(t *testing.T) {
t.SkipNow()
ctx := context.Background()
address, cleanup := test.CreateAPIWithDependencies(t, ctx, "queryService")
@@ -19,17 +20,50 @@ func TestJobCollectorService(t *testing.T) {
client, err := queryservice.NewClientWithResponses(address)
assert.Nil(t, err)
idRes, err := client.CreateJobCollectorWithResponse(ctx, queryservice.JobCollectorCreate{})
contextRes, err := client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.CONTEXTFULL,
})
assert.Nil(t, err)
assert.NotNil(t, idRes)
id := uuid.New()
collRes, err := client.GetJobCollectorByIdWithResponse(ctx, id.String())
jsoncfg := "{\"path\":\"key\"}"
jsonRes, err := client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.JSONEXTRACTOR,
Config: &jsoncfg,
RequiredQueries: &[]string{
contextRes.JSON201.Id,
},
})
assert.Nil(t, err)
assert.NotNil(t, collRes)
res, err := client.UpdateJobCollectorWithResponse(ctx, id.String(), queryservice.JobCollectorUpdate{})
jobID := uuid.NewString()
fields := []queryservice.JobCollectorField{
{
Name: "json_output",
QueryId: jsonRes.JSON201.Id,
},
}
collRes, err := client.GetJobCollectorByJobIdWithResponse(ctx, jobID)
assert.Nil(t, err)
assert.NotNil(t, collRes.JSON200)
assert.EqualExportedValues(t, queryservice.JobCollector{
JobId: jobID,
Fields: fields,
}, *collRes.JSON200)
fields = []queryservice.JobCollectorField{
{
Name: "json",
QueryId: jsonRes.JSON201.Id,
},
}
res, err := client.UpdateJobCollectorByJobIdWithResponse(ctx, jobID, queryservice.JobCollectorUpdate{
Fields: &fields,
})
assert.Nil(t, err)
assert.EqualExportedValues(t, queryservice.JobCollector{
JobId: jobID,
Fields: fields,
}, *collRes.JSON200)
assert.NotNil(t, res)
}