diff --git a/CLAUDE.MD b/CLAUDE.MD
index cb0a1ef5..46a32814 100644
--- a/CLAUDE.MD
+++ b/CLAUDE.MD
@@ -61,11 +61,11 @@ Use `task go:lint -- --fix` to fix go (file not formmatted) types of errors from
### System Design
-This is a **document processing and query orchestration platform** with microservices architecture using:
+This is a **document processing platform** with microservices architecture using:
- **Event-driven processing** via AWS SQS queues
- **RESTful API** (queryAPI on port 8080) for external interactions
-- **Runner services** (ports 8081-8090) as queue consumers
+- **Runner services** (ports 8081-8089) as queue consumers
- **PostgreSQL** with migration-driven schema management
- **AWS services** (S3, SQS, Textract, Cognito) for cloud functionality
@@ -76,15 +76,14 @@ This is a **document processing and query orchestration platform** with microser
3. **Sync** → Client eligibility → docSyncRunner
4. **Clean** → Content processing → docCleanRunner
5. **Text Extraction** → AWS Textract OCR → docTextRunner
-6. **Query Processing** → Custom queries → queryRunner
-7. **Export** → Result generation
+6. **Export** → Result generation
### Core Entities
- **Client**: Customer configurations with document sets and permissions
-- **Collector**: Query collection specifications per client
-- **Query**: Data processing blocks with dependency chains
+- **Collector**: Processing configuration specifications per client
- **Document**: Uploaded files with processing state tracking
+- **Folder/Label**: Document organization structures
- **Export**: Manual result output processes
### Service Architecture
diff --git a/api/docTextRunner/runner_test.go b/api/docTextRunner/runner_test.go
index 8ca6444e..b3788734 100644
--- a/api/docTextRunner/runner_test.go
+++ b/api/docTextRunner/runner_test.go
@@ -1,9 +1,7 @@
package doctextrunner_test
import (
- "fmt"
"io"
- "regexp"
"strings"
"testing"
"time"
@@ -14,7 +12,6 @@ import (
"queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/aws"
"queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/textract"
"queryorchestration/internal/test"
objectstoremock "queryorchestration/mocks/objectstore"
@@ -28,10 +25,10 @@ import (
"github.com/stretchr/testify/require"
)
+// Note: Query sync functionality has been removed. See remove_query_plan.md for details.
type DocTextConfig struct {
runner.BaseConfig[doctextrunner.Body]
aws.AWSConfig
- querysync.QuerySyncConfig
textract.TextractConfig
objectstore.ObjectStoreConfig
}
@@ -42,7 +39,6 @@ func TestDocTextRunner(t *testing.T) {
acfg := test.CreateAWSContainer(t, cfg)
test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
- cfg.QuerySyncURL = test.CreateQueue(t, cfg, test.QuerySyncRunnerName)
mockStore := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockStore
@@ -135,9 +131,8 @@ func TestDocTextRunner(t *testing.T) {
doc := doctextrunner.Body{
DocumentID: docId,
}
+ // Note: Query sync functionality has been removed. Text extraction is now the terminal step.
assert.True(t, runner.Process(t.Context(), doc))
-
- test.AssertMessageBody(t, cfg, cfg.GetQuerySyncURL(), regexp.MustCompile(fmt.Sprintf("{\"id\":\"%s\"}", docId.String())))
}
const pdfHelloWorld = `%PDF-1.4
diff --git a/api/queryAPI/api.gen.go b/api/queryAPI/api.gen.go
index 67604607..e23c734d 100644
--- a/api/queryAPI/api.gen.go
+++ b/api/queryAPI/api.gen.go
@@ -66,12 +66,6 @@ const (
RightClosedInterval FieldFilterCondition = "right_closed_interval"
)
-// Defines values for QueryType.
-const (
- CONTEXTFULL QueryType = "CONTEXT_FULL"
- JSONEXTRACTOR QueryType = "JSON_EXTRACTOR"
-)
-
// Defines values for ListAdminUsersParamsStatus.
const (
All ListAdminUsersParamsStatus = "all"
@@ -521,9 +515,6 @@ type Collector struct {
// ClientId The client external id
ClientId ClientID `json:"client_id"`
- // Fields The fields in the collector.
- Fields CollectorFields `json:"fields"`
-
// LatestVersion The desired version.
LatestVersion Version `json:"latest_version"`
@@ -534,29 +525,14 @@ type Collector struct {
MinimumTextVersion CodeVersion `json:"minimum_text_version"`
}
-// CollectorField The field properties for the collector.
-type CollectorField struct {
- // Name The output field name.
- Name CollectorFieldName `json:"name"`
-
- // QueryId The query id.
- QueryId QueryID `json:"query_id"`
-}
-
// CollectorFieldName The output field name.
type CollectorFieldName = string
-// CollectorFields The fields in the collector.
-type CollectorFields = []CollectorField
-
// CollectorSet Payload for updating a Collector.
type CollectorSet struct {
// ActiveVersion The desired version.
ActiveVersion *Version `json:"active_version,omitempty"`
- // Fields The fields in the collector.
- Fields *CollectorFields `json:"fields,omitempty"`
-
// MinimumCleanerVersion The desired code version.
MinimumCleanerVersion *CodeVersion `json:"minimum_cleaner_version,omitempty"`
@@ -581,9 +557,6 @@ type DocumentEnriched struct {
// ClientId The client external id
ClientId ClientID `json:"client_id"`
- // Fields The fields and the value for the document
- Fields map[string]interface{} `json:"fields"`
-
// FileSizeBytes File size in bytes (null for legacy documents)
FileSizeBytes *int64 `json:"fileSizeBytes,omitempty"`
@@ -900,84 +873,6 @@ type LabelRecord struct {
// ListDocuments The documents in the client.
type ListDocuments = []DocumentSummary
-// ListQueries A set of queries.
-type ListQueries struct {
- // Queries List of queries.
- Queries []Query `json:"queries"`
-}
-
-// Query A logic unit of execution.
-type Query struct {
- // ActiveVersion The desired version.
- ActiveVersion Version `json:"active_version"`
-
- // Config Configuration for the query.
- Config *QueryConfig `json:"config,omitempty"`
-
- // Id The query id.
- Id QueryID `json:"id"`
-
- // LatestVersion The desired version.
- LatestVersion Version `json:"latest_version"`
-
- // RequiredQueries List of required query IDs.
- RequiredQueries *RequiredQueryIDs `json:"required_queries,omitempty"`
-
- // Type Specifies the type of the query.
- Type QueryType `json:"type"`
-}
-
-// QueryConfig Configuration for the query.
-type QueryConfig = string
-
-// QueryCreate The parameters required to create a query.
-type QueryCreate struct {
- // Config Configuration for the query.
- Config *QueryConfig `json:"config,omitempty"`
-
- // RequiredQueries List of required query IDs.
- RequiredQueries *RequiredQueryIDs `json:"required_queries,omitempty"`
-
- // Type Specifies the type of the query.
- Type QueryType `json:"type"`
-}
-
-// QueryID The query id.
-type QueryID = openapi_types.UUID
-
-// QueryTestRequest The properties for a query test request.
-type QueryTestRequest struct {
- // DocumentId The document id.
- DocumentId DocumentID `json:"document_id"`
-
- // QueryVersion The desired version.
- QueryVersion Version `json:"query_version"`
-}
-
-// QueryTestResponse The response from a query test.
-type QueryTestResponse struct {
- // Value Result of the query test.
- Value string `json:"value"`
-}
-
-// QueryType Specifies the type of the query.
-type QueryType string
-
-// QueryUpdate The properties that may be updated for a query.
-type QueryUpdate struct {
- // ActiveVersion The desired version.
- ActiveVersion *Version `json:"active_version,omitempty"`
-
- // Config Configuration for the query.
- Config *QueryConfig `json:"config,omitempty"`
-
- // RequiredQueries List of required query IDs.
- RequiredQueries *RequiredQueryIDs `json:"required_queries,omitempty"`
-}
-
-// RequiredQueryIDs List of required query IDs.
-type RequiredQueryIDs = []QueryID
-
// SingleFields Single-value fields for a document extraction (1:1 relationship)
type SingleFields struct {
// AareteDerivedAmendmentNum Amendment number derived by Aarete
@@ -1216,15 +1111,6 @@ type CreateFolderJSONRequestBody = FolderCreate
// RenameFolderJSONRequestBody defines body for RenameFolder for application/json ContentType.
type RenameFolderJSONRequestBody = FolderRename
-// CreateQueryJSONRequestBody defines body for CreateQuery for application/json ContentType.
-type CreateQueryJSONRequestBody = QueryCreate
-
-// UpdateQueryJSONRequestBody defines body for UpdateQuery for application/json ContentType.
-type UpdateQueryJSONRequestBody = QueryUpdate
-
-// TestQueryJSONRequestBody defines body for TestQuery for application/json ContentType.
-type TestQueryJSONRequestBody = QueryTestRequest
-
// ServerInterface represents all server handlers.
type ServerInterface interface {
// List users
@@ -1350,21 +1236,6 @@ type ServerInterface interface {
// Logout from the application
// (GET /logout)
Logout(ctx echo.Context) error
- // List queries
- // (GET /query)
- ListQueries(ctx echo.Context) error
- // Create a new query
- // (POST /query)
- CreateQuery(ctx echo.Context) error
- // Get a query by ID
- // (GET /query/{id})
- GetQuery(ctx echo.Context, id QueryID) error
- // Update a query
- // (PATCH /query/{id})
- UpdateQuery(ctx echo.Context, id QueryID) error
- // Test a query
- // (POST /query/{id}/test)
- TestQuery(ctx echo.Context, id QueryID) error
}
// ServerInterfaceWrapper converts echo contexts to parameters.
@@ -2181,82 +2052,6 @@ func (w *ServerInterfaceWrapper) Logout(ctx echo.Context) error {
return err
}
-// ListQueries converts echo context to params.
-func (w *ServerInterfaceWrapper) ListQueries(ctx echo.Context) error {
- var err error
-
- ctx.Set(JwtAuthScopes, []string{})
-
- // Invoke the callback with all the unmarshaled arguments
- err = w.Handler.ListQueries(ctx)
- return err
-}
-
-// CreateQuery converts echo context to params.
-func (w *ServerInterfaceWrapper) CreateQuery(ctx echo.Context) error {
- var err error
-
- ctx.Set(JwtAuthScopes, []string{})
-
- // Invoke the callback with all the unmarshaled arguments
- err = w.Handler.CreateQuery(ctx)
- return err
-}
-
-// GetQuery converts echo context to params.
-func (w *ServerInterfaceWrapper) GetQuery(ctx echo.Context) error {
- var err error
- // ------------- Path parameter "id" -------------
- var id QueryID
-
- err = runtime.BindStyledParameterWithOptions("simple", "id", ctx.Param("id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true})
- if err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter id: %s", err))
- }
-
- ctx.Set(JwtAuthScopes, []string{})
-
- // Invoke the callback with all the unmarshaled arguments
- err = w.Handler.GetQuery(ctx, id)
- return err
-}
-
-// UpdateQuery converts echo context to params.
-func (w *ServerInterfaceWrapper) UpdateQuery(ctx echo.Context) error {
- var err error
- // ------------- Path parameter "id" -------------
- var id QueryID
-
- err = runtime.BindStyledParameterWithOptions("simple", "id", ctx.Param("id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true})
- if err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter id: %s", err))
- }
-
- ctx.Set(JwtAuthScopes, []string{})
-
- // Invoke the callback with all the unmarshaled arguments
- err = w.Handler.UpdateQuery(ctx, id)
- return err
-}
-
-// TestQuery converts echo context to params.
-func (w *ServerInterfaceWrapper) TestQuery(ctx echo.Context) error {
- var err error
- // ------------- Path parameter "id" -------------
- var id QueryID
-
- err = runtime.BindStyledParameterWithOptions("simple", "id", ctx.Param("id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true})
- if err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter id: %s", err))
- }
-
- ctx.Set(JwtAuthScopes, []string{})
-
- // Invoke the callback with all the unmarshaled arguments
- err = w.Handler.TestQuery(ctx, id)
- return err
-}
-
// This is a simple interface which specifies echo.Route addition functions which
// are present on both echo.Echo and echo.Group, since we want to allow using
// either of them for path registration
@@ -2326,244 +2121,226 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL
router.GET(baseURL+"/login", wrapper.Login)
router.GET(baseURL+"/login-callback", wrapper.LoginCallback)
router.GET(baseURL+"/logout", wrapper.Logout)
- router.GET(baseURL+"/query", wrapper.ListQueries)
- router.POST(baseURL+"/query", wrapper.CreateQuery)
- router.GET(baseURL+"/query/:id", wrapper.GetQuery)
- router.PATCH(baseURL+"/query/:id", wrapper.UpdateQuery)
- router.POST(baseURL+"/query/:id/test", wrapper.TestQuery)
}
// Base64 encoded, gzipped, json marshaled Swagger object
var swaggerSpec = []string{
- "H4sIAAAAAAAC/+y9C3Mbt7Iw+FdQ3LsVOeFDoiQ/tHVrr6JHorO2pSPJm3tP5KsCZ0AS8RBgAIxk2uX/",
- "/hUawAxmBjMcyqTs5OjUqYrFwaPR6Bcaje7PnYjP5pwRpmTn4HNnSnBMBPzzEivyms6o0n/EREaCzhXl",
- "rHMAn1CivyHKxlzMsP6AKEPmD3TTga//zz1lMb//T0VnpGf+fdPpdDvkI57NE9I56Oxsb7tGO9udbkdG",
- "UzLDesZgm+e6zQx/fE3YRE07B7vDbmeOlSJCg/W/v2/3Xr3/yTU2f/1Hp9tRi7keSCpB2aTz5csX3Uvg",
- "GVF2rT9jFU3PjqsrvZ4SNNIfUTpPOI7R2XG/0+1Q/W2O1bTT7TA804NDq1sad7odQf5MqSBx50CJlPiL",
- "+g9Bxp2Dzv81yLE+MF/lwMGgoTtKKGGqDqAIvtaD8hVAZBNrKI55lM4a4Ijt941A4k2uYTn5OOeiFhIC",
- "XzcCRzaxhuKfKRGLOiDOjhEfIzUl6E/dbP2guNk1JO8kESczTJMqLO8uX/cIi3hMYpRKIhDR7RCOY0Gk",
- "rAEL2jRCZljba1rgw+0glwki55xJYpks/gUrco8X+q+IM0UYCBc8nyc0Ahky+EPqNXxuuzlCcPGGSIkn",
- "xMxYRMXJRy0acIIkEXc0IojoDhoFdYIuNJttO8gbwlRHnI0TGqlHW80lkTwVEUE4EQTHC0Q+Uqkk4gJJ",
- "pUVyZCFa0wJPuRjROCbs0VZ4xmQ6HtMI5NuciBmVknImkeL6T02CSE2pRDjSPda0zjNmqASge8S1erSp",
- "uXSNpHnG7nBC40vyZ0qkesQlwbRImHnRiMeLNa3oLVenPGXx4zMb4wqN9dRrWsk1528wW9i9kY+5IJDF",
- "KJ1zhhTnaIbZwu2VXMPqup1LosSidzhWRFT10tt0NiJC60hJIs5iiUZkzAVBSiwomyA8wRRYOjP9drRa",
- "CWggytTu0GggOktnnYOXz/e2t7udGWXm71wbUabIhAiNEK01GU7VlAv6icSPj/j7KWEo9UBYC0V9cSiC",
- "MQ7jGWXaODgECenmDtjwDqoxF9ZMYHiUkEFMpf6vFbES3VM1RTKNIiIl6JlUIsxipI16qfBs3ul25oLP",
- "iVDUKHrTszqlAckJcqINIcL0dv3esZPCL/CP9/4pIf9asjG6nYhPGFX8VqajP0iktAFemffItEG2DaIx",
- "YYqOKRGweG2xGZCRs1cKZxS8MxpGu/Fej+yPn/devHy13cOjKO6R8c5wd2//uf6laA4N958XziX90Cmk",
- "aw2pCrhg1jlTEnYGR0rbcnPOCoD9waesH3PyX/anfsRnne7Khlq3Y3e3CspvU6KmpIAiaEtis3sOFGMr",
- "2nFHnCcEMz1wTiNVi9l9ciu1VOMvcLg93O/tbPd2nl/v7B0M9w92d//lLzDGivT0HMVF7geMUd+y/T1b",
- "cDfDkZ39fdaTA63oRWQsdSQIVkFWytUdUFSkG4JIQ4zcmz0ENoLZukjb3F1gIsE1p0lJJ2wGZ/AyL9XQ",
- "yLuKXY+2UklihCVy5K6n1TM9WwvR+Mds3Pt02PvXdu9V//b//unmpvf+p//yfoMfbm769qf3n4fdL0H6",
- "H1Mh1a05gIQW+INEE3pHGOALEIujiKdMWQSXqOUffMqKgO9YpZD93YYrE7wMqDGe0WTREqpjTtYAlKaT",
- "AH++plJp/gEy+kAWYCgbckKUoQttQas+5WiL9Cf9LpLpnIhbrAm6C/Th/o3TmCouuta3cKu/PeujSz0t",
- "mqVSmbOGHnTBU+GNTNgdFRyot48yAxD6jTV5SpoQppIF6qFoSqIP5tutAZLEekQndQu6//dODp/mTwOg",
- "ZlCqyAxwUUbqcjTO8Mcz03vfbIL9aydri4XAi4q8cEzhUaxPKG5/WkiPFgoZa/PJlyEaEV1EWZSksf6l",
- "rM/OjkGWyAWLrIKuiJFWahIAND6DHzwxwuifKQkoz8dRkpGB6harJh0CplWmL++xRLafpi/9u1xIRWZN",
- "2mX3YG//gdql215Mb0aBG6Mpblbgkbe/iEpr7wGG7F63UelNQrtEQbnwbiOnv0oyl2b2JHQLYbx8ZnBF",
- "qFvNYU1YzojP2hfjNEkWPiXmYtMdAozn3FBnH52N0RgnknStRW78O/kGoa0IM+irmTDCijxDo1TBGbUo",
- "76dYIlaaZuB5VJ710RvMUpwYscEFEvrshmZ4gUYEOeHXb0MSRYkekGygDdQUK3RPBClix1cEGfwaOIAD",
- "a7kjiVFyxswiTr3QMZJcq2BMExL30RGfzbEgxtIqNwa5rrVjTJQWY7nNBf1TQSSgn6VJorFBZnO16GoU",
- "mv4AeAbrFmyPxXERq+csWaC5IFIPTceoQDqa7TQWH1vTFZVbt2O1RO1JCZaXqZKcf07PL49Obo9+PXz7",
- "y8ntxeHV1W/nl8dV8bgMvJJ2DSin3DDPFVqBBxtV7TFJSCtVm519Y91Dc6I57ZpT7ljwGSI4mlr+RFsO",
- "P1rdZsT67GHatv5Qag9EAJOVaY+ja+2Mt3rhVYiPHY5q0FOHhhby0i0VhlxBGRmaWHWCbOeWTxGm1Iwa",
- "g2TY5lhfu7ePe6bPyH5jp3o3Q5Pl9eJgZ39T5/oCSS8RGgrTJCgUNdjK2pb+hbdPrkWZ4FnrVv/4HrOH",
- "2efvWtrlBizPSMgtjO/XZHdnaM90D5PM/vX2q4Od/YPd7XUb6ycFd0rR/wbAPbLtfpQKoY0IZ6dbsesD",
- "tj7Tvepv0fYgJXcNUnkjlnzAx9ICkoca9jV+FWOxBk3TouXmjLY7Su7JN7LZLJkUbDfHTgEb7uj87enZ",
- "5ZuTB9ht3U46j1uwNh8jvcPItG4S/dsHO9vrEf3tjEjHbI2a4DWVgeVd4AllcJBLrNdNY9rejyiucIIM",
- "zrWon5vGWvLOiMIxVrgi86dY3s64IPUqWn/VQ8H5hSB8h2nirkmWmkZ4QuqJRX9FDC7kChdu3YZLtp3t",
- "7cIl2071ks1MeyvpJ9J0/2cQNycC4PAB0BKyEYJl88NGBGgT9oeVIJhhFU21jh7TRJUwsfeiCZLhzt6L",
- "vZe7z3WrplvHLpztGjy3BhC4NPC2puPJkaarx4rxUpAigK5Gv6YBzmHNEo2/iY188s6w9/L7EJADGs9G",
- "Nikl6ChVRKItUEn2PgRERvFqRFbPVo1KzEin8q2BU5cl3cWwmuJN3xhYiMpXBkGQjjnpXc0ohGVt6MIg",
- "s2WTtjcHPXR5cvH68OjkCuEkMS6p3KzdYlxpk4kqekee9dE1138hDJ+dz5pAT0ffpuM8SSVggZF7xBmB",
- "roLM+B3JenMIMFVG68MVoSbi8K3E11xIuOsH24gLNKYQmKOwIn10rsGAqKMxJUmMtFUtSALepITgO2JH",
- "T1k0xWxScpp983uMKv/qT6d6KbpXlUauKJskelH3mkCM48ysXKsxtLWzMzR/A65mmhg4g/ZVbsVYEEWO",
- "iaB3JD4EOuHsEityBKjSJkNCGSlhYbgdMpQLYx0lmM6uF3NyFIdQ2Nz5lJCraEriNHnAzF7n/58IaSMd",
- "VgPgF8HTORHh/i1geM1Hq3d6S9Q9Fx9W73gheJyaOMeVO04Enj2o4901tFi15yWhs9EboqY8btfZ0uQx",
- "kVF1H5t6vMEfTwnRtHzGokJ4UsxTY6Rl5sKr7H/9V6+CFoO1xYrjX0Sq9fh9PfqyoVM11dpbadwGVrwf",
- "XPCIJskKnJY3bz1DhMUd4alqOYNrfmbCAau2b6SFQ7bQtqPqTldwUHhIj/aLnau9C8GjtpNkzVee4Q1f",
- "bYo3vP0cMRnjNKnfgpjiScsVmqbtZ5ZTy3drZLpYTuuXIqeWEdfDheTjlI6ourAHtKX4se2vqaqoqzCG",
- "xjiiCVWLw/iPVCptCF0TMWvVdTIj60fuZEZqkTuZkbUidwK+Q3E+rp/R6N7DBMKvtd36mtyR5Hx8hEU7",
- "hWNH+BnLtWPKjNyScbLWrXnH9vgVR8vQs+ZNgTEvibaCzU3psvkvqfxwPn7DhcKalK/SUZRgKVfByxW5",
- "I4KqxUP6LAPvWmAmx0QsbVc1YsLz000wHm1gPLpmxkuIlI18l/CRNSQvSQKuMTml81YMZ7pqU3Llrixu",
- "q2ihZWtOYgrP179hetRaBOqPa90ynqqEEnHyMUpSzZQtEVXt1hprtuspFVId8yTB9QyUNf2ojztSvh2J",
- "Y7yQ11NB5JQncUus99tiIZto1RlWmOON67uhgU8h6oVFi3UjZxNk19ommWNFCVOHE/IGf6z0aO5AWasO",
- "CY7I+fjKPBJsyQblTq2ZYC743dWcRBQnatF2smKflea6xh8547NVpsq7tJ5JYDYhlklbzSOwIicywglW",
- "XKwwjderVngUWr3BH+0p+sI4Mb6ehAvjh91Za2E/QehsdMTZnXEUneJImReK7UZvN/7JeEwibQcfl7Bj",
- "7u6WMQ+MsX5VCMOuVe6YEQW/e2td9cuJLesxpxVro6n9NWWt27eWg1lre7P40O26IyxtK+Sy1q0Z1L60",
- "br0qqfhcq91jTJOFZdU1kpEbflUjp9KvPQJszzZGTt52g8aHm2T91kdp5PWbH24CKwfO2cnHiEh5NMVi",
- "Yq8Dv1osuDkuSwpsrQtozQ5K0MnE+BY2QQt2+CM8z0Y/nKk1TpBG61cFab23Io3WqiJSRtX5+A3BMhVt",
- "zu2h67X2uWRo8aaws73zav/ldjzukfHz/d6L5y+e917G5FXv1d7e7j5+tbu7+wL7sTJpCkEufoTZ8wBJ",
- "AUBXNdFDGqg8sAwXAPTe584F14ynR+xCyiCIqtTAQPC9/hGziCTF2JoSDO9gVC/OEifJ+bhz8HuLxDim",
- "71U6m2Gx6Hzpfi6HBQAct2OaEIZnTW/lsibmRYLpiAoL9C5vYx7t9ufxuNPV/9x5Af+uubzd2371fLXb",
- "WxtSs+zO9n23EoqtDNh+NCoe8VSVttCLQ7ULzTFU3JZw0JNDmj+muSLH9qlg5coXmpp/tophCW5vcxSL",
- "DVi5NVGZS4N9LEQ28ILKHPI87mh/HdE+pegah4kiuO/rZIbBQYtXC4KoVDASmxDa8oZLhKOIzCHqhIsi",
- "YQd2yoYat8pM5QchLu1gJU7W6TYV4WRFSHEbiWEWEohZHJgdGxweHg5c1qkBtB6sLDQF7QkyJoKwqGwv",
- "7wyXRRl6Wb4yML3lLdlbR9/VkAvzYSk7r2ML7Qvf5X3yXGCeyA9Gfv7mQrkLlJj1KdJgm1BPliYm0LEY",
- "3tguxDwMTTGw/EFx41bHOAKUTSGOWaMWWma4nkhDamKdbvWXdkAyznoXx6egE6RJcoMoQ1hEU3pHWgeG",
- "tgeRCzqhDCeZnq5Cd26bZIoqC3iHfSQx+tfZRQjETrbk/ic6L25nW81s92f1Tc46IsnRGBfi+vbWtL9z",
- "wSeCSHk7JyIiIcV3kZEXco2RbVyKs22GaWdZMp0HKgKjBhtQW9bcOYKpx9IFwtze3pTmNmI+F5ch8vXU",
- "QHlxYWoKSJFazg1seUHyhbSNEdpHmF0tTNRQKUOX4SWbQVLbCknC7yHNALgA4TWw7IQCyu3INTlQ9Dki",
- "V01eFhSTpa2otVbTPU5OLG8Pjr3yXsLOwRj1+FqSaZO4NIK0+Cjp8PCwev5aLmXcnD/zeNE4rzkgPhx3",
- "VUzUo0Bb+i1sz4gzhSkziSlcEDGE98Iosgqw/VB/trANipka+m1D4I95ZBbQ4mRlmfSo3bHBW9HaBYxD",
- "SgGi+s15G1SVHqVUMh0cHgpS9ge3fL5npqxzGMAt1JgSE73tuw4MKH3Pa3D29vbqf94edbqdt+fX8E94",
- "deT+OHv7S9BZ4AMQ5pFDt247v283G9xV6bCdxiosvvK81Pxcv091LyJK0hFMQptpwb6oCvANZvDYvR3E",
- "TuI/TF5Wl8NjP7o5kPuXSI0VFPGYIHs9VeaU53sFF9xwuLv7Yri9+/zl/t6LF89bPCY64klC3JVX+SGD",
- "/YRmPCZJP5iE7o7c3uWLaMKHW+tDz0cmHn5pDwf0qWkOb0cUkeoBYFrM3UZaPxPRdgR/Y71RFPmoHjRE",
- "UK4Za8nipB7UmukrSOmWNzPIgAXchonWvGIomSlgD7nOVTpqxU2FuQ1XdTuQBboFHflpnX1cWpmeDbN8",
- "1fVqgqdqniqLAD1w/2GqoUTA9VjO1HkBta00emknC2od4Cwr9azDFQm+21yAA6DwEA2jo/otf7DoeLAU",
- "+H6YuUJhuX21TKuVjIA1abNvdFbo5hC/D2MFDmknTNBoGnq9777kWfJjc/FhHgy7Vx44qX8g/JWaqJYz",
- "IRfFlKA7nKT5c0QHp29GZrcsi51MDphuObCLoXEnuD93jdMuREv62HxFP5GfFyp0Q3NKE4Ik/US07Bgt",
- "4IEo5GPSICZkgqNF7hF41mhs7A1f7b16/mL4an+ZI+MBnigPUxUfU9VpyJOYiLM4JJjgPaRpgM6Ouyb5",
- "FM3dHvqAblN7Yduu+f4v6LacYnlNPqpLEnHRkLwMIy039FlX2FSvW/a9Y/bLMyRgEJeYLLtQ8TBSdRxM",
- "sZwuo+BfdZtW3O7XhdCWwoiEcrYcJgky3xAkdiYxUrwCayt19FoPY5FXOWJWlZHzEV1gNW2gqTlWUy08",
- "72isZUQKnGVd1Vt0NksVHiWQunU5ianC7jatBTTOSbaf2Xk/KAd9Ww720DPpijSV7UOTqFxaRaT2Ony0",
- "O37Ze7G797L3Co92ejtkf3+0O955SeIXD7gOz+Bhp4ajqomuHERaUnu8Z3P4OsHu5DbaSuhkqkw26RlW",
- "XUQ+wrNnd9F4S2OQuQZ7oSftT2KxXiw+psR7Em//PuLNirQHSLLaa9yCOJP2Tjfn9X4oBcyGaKd2xaGF",
- "FSoYBPL+ZX9pVoc6JUUjcea6dhY8NYkZsswPyH38Ul78rG5CCwkyv4406cBJEqelHNH5dBN6R9CMoAnn",
- "haCYB5xyS5hzUAbxBrWhahPH+SdPW6HKxt8VUl6vweg3x/vbhJtsb2HKdF+1Grmf0mgKWLWAfaJzEOjo",
- "niYJGpG85EmObbl7MBjgweHA9BkMt4f728Od3YEWy/1I3pWwvb33soBu6N//Sf/fjgAl2j6//DLo/3Rz",
- "o0cIeh3aeW7NZtR4bn1bpsGL26rKWI2dQnp78f5O78XObtR79ZKQ3t7zPfJy9+XOznhn9wF2SmE94bM3",
- "l5KOkgwwyFcifQe8H6lH2a27Tszj9t7XTnxtKLWZqi05g0eFWTD6AfuGJBCap4KpkE7hg7lOiPicoBGW",
- "JEacWXeV9V3BsVO29iKBLjBDt3AhUTYhUsPzEDCzziivKljFAmHxbf3tACQ/0p+1iM0G7LcIVdkdhllG",
- "qNrpIJWNWOOE1cN+6PhfUc9Zdapw/ijFbcSOraNRPo7aZL0mXY02sL1cNdUUmzhLexMyqKAnRIeW5gBi",
- "Q1s4SYymmeI7uAWVVCpzAyXSSKUCjJl2SbuK+XfaXFvaS/+fFw1pZSG3VlY1wIh2t4qCvNINvzprpTNz",
- "zla0aM1unbZylF75bcsS3QOgNGq3sNU+8t63Icq6K/DTIPVZN2vJ5lgf5W2eqA5VTTp9DYNf5GkT+XtX",
- "IO37Kc9S6cNJ53sk71De3grhwJl2nWH/QcQ+nNO6nbu6q197VeCiJLZ2eqAFn7WOFSw9wthZGikBay0w",
- "e34t2IbtD9WqIqD23ruykY753VFPm0RTKhUXi6pd/xfnthzrT6xWXsJfhF1W5AhrPNcYwuYWNeJJOmOB",
- "dObMXG+tYKUfZX3cLdbtw+/ezWEhbGqbb5rMx2li/GbEZoAtnC5Wvx5fJfWqt8Kuh68M9CXbcuRjuOFo",
- "aKyKcbZl2UyFc2JCpLy12VBtBiX3Z5RwSeJbTW/iDpLE8jlh/t8JGavbajNBJ9PQ79YzBCxg/hU6hNb5",
- "5s3vxSz8XCAuJpjRT3qFpfjZimvlbLWHD38ZsZ1XvyqL7uzmcM2SOyiBPbf9xuXujChBoxWeEhro3phu",
- "ZwxyNVRf13lh9HaG/CrA4NKV+eHzNAE8w0ss2/g/lUjJDaMSSRsrmVUhs1WJ+ugtlBlSUyLuqST9G81p",
- "xUcvkMdEZPS69I5kJWSPliN76ROcefAGwO4/+P+34MAMrghzWBpASUGeAT7F0rS86QxuOkWvWvbOSw6G",
- "28O9SkrkYp7cQf/H5a5cWKYte5/JgpXUIoC9rAxoxW1hFvsYYmpVu45sTDbU0+753Ma9zL+SiB8iMR5I",
- "tO8kMTQKuyhyEt44zdaSayOJNj/mNaAvecZrGy0dpY9OcDR1u2iVu0RUSQhp0fB3bUkEQxH9G6ZlQC8h",
- "dyTJYAG3mmuCJAE20jKoj47s+4AIM03DNyxrpjjc2DLjg/PoGU0pEVhE08VACUJyJ52RtO0cyIYzWr3X",
- "Lhh2FnH1u/MmV1utFA8uYteXHqbkRFWGjBZwrwuaMYs0uyhtcGOhh+VPwGqCJgxEaLQwN9TFm8JjLKcj",
- "jkV8e0lwvLCG8hm4nzX6djTXnh9d3l64t1NQBd5sgg+/b2B7AWa1UQXvTLWkQOnvgATckPCBpx7HD3z9",
- "VgUTcPWVuxikXRAzJWC7GUktpWtrV1XfnsHv3hvYMJFvZdEzDhokKYsIouoHicwlC2Hq2RPNG9x//1T1",
- "AFK6rAk+8mwsYYKP4N7R1qTIVlMkjLDSf5ubZutW/D0DW7w2AyCEqF9t7EhDCIoNcWm6k6/xZJzFteEg",
- "h+7GDW79a94nthG++g+16K98Nd7uVeNZbMZf7U2jO0T/IP26fVNChV/mtbLoFuXZvXJi1uVk7HGoHhWY",
- "oJWNki2SJ2RVB1R9yfLCqIH9h5ox9avRpl8TsgrD1Z4O4mLUEZyeebH0VUfL0x7XJ3EMdR81LHeU3Hsy",
- "Tu+fiUuQZW4c7gUI/wOpietyhXIGdQXQbWGX4iTDIrf9jnuftnuvbnvvw2wXFnq/pjPMeoLgWB/IDSTV",
- "B5+h+Vv6Lf3dqiVirxGaCMz0CXK0MF6R8s7k17MHGmzPJX2Q6KOJbqr3xHytq4VTj7yDZYj0C+QsYwS9",
- "5dmrDx8VId4AxXU4nye0LtTKU1F4Pte0ie65+DBO+L2xDuCLH6dZui02AZ3Lj/IwuhNaFbtjPcf4xFlU",
- "JZqAdYD+3cLJfIpZOiOCRsBuKdPnn4gLgjRvFi47SnZOY8nG3w97/7Lb/L5FoJ7DQI6/2v2rCww2v5uH",
- "TGarvOja5Vt2uLzeqRn1HmdxuxtzGbcho4LLOFuqZugNkdM67/oMDdqI7cfxHtVww29F9v6+2KJ6IVhl",
- "lK5HwUGmoVI1nTA8mzN/95k9BGybxqEQW708ZE8D9c+UiJBVoY1UAlrrT9Oiaqf+WdfV6TuvZ6sVwDPe",
- "ku7xEoqFdY+DIoR1M2BgaQmf0AiljAKc5COJ0nCelYe/gedsTCetVnxkmrYKUM9eOn/Fu3eHvVtvA5u6",
- "Xtr2dm6Th8hW5VgKK5TvCHIUDFF5mF5ZV+2+HmUYLuc10L+nIr/a1MwEb8D7S45yNzef+z/e3HwJGkRm",
- "0qbkPVnoKnKLLdxkZCBUbvpXp5Rvv4fQu3Zz6gLAAQePlbbVAE+kqo2TDeRcsvuENBFmt43VE5gVtrer",
- "vkcyuQhWZdqayE0TjV8csnZLDCLqjvJwQHPHeVP+3kNEFQHmCXPII5AmyllGxQFWdqKUVm2mrF+fJeem",
- "ZDu6ZwE4P4TjH1fnb29P/vv68vDo+vyy0+0cnb+9Pvnv69vTd69fB0MsYN6HZ6zx6e1bq56vFyihePVK",
- "q3oHj5OYhmbOjlc0Gwx3LTF4rkoBnaGqrT37pj8rz+odWwpPu3cOdpDwiiktK906IyzWg7xNAxVjs6/O",
- "zRybXmi0QIfYJqJqDHJb5ncvAFMqWFFRoLBKRFwrFCpCv9fb3ult75RPV20OVj4olXIMNcCovB2qqYk/",
- "7O0+CJxU8UvCyD1ObJb00t6kiveEaYAoi2mEjZuo+pLVG8rlq28YSy9JlgVjuExmfQqzt9774ewienmB",
- "S4vWrCRi+cytknxU27bNuGOakLfLHz1XnW9ukltNWTZdeKvZ9Ei2WEhpDfgjOrOeRvtsz+YdBfXmJaBc",
- "VmRkjhdELMc+NGsDNjS8UmGtcc97CdF60TxGM+nBIA21HT7H2VEpd+Kwcsp9/3n4pS5N6x2UUNYLOE2T",
- "wJlc/4rc63T7EFkgqMrXapHZDKbyS+We3hsPvb04a7MPbsjgfpeGvD5723LIczU129sCDxBwlmGjLR7M",
- "DCE8nBfGWwEP0DGIh9KQ7fGgm69OlTkqcsJ8+z8PJMyQDdEqiV5N/rwHZ5qUJEoFVYsrbWO4c9qEUcUP",
- "09BVqP5VixoraFK4nD+c4U+coSPTEZ0dX/TRmUaRcfNcnh69fDHcR//47RqN9GljDrZFZA8iDgRYU8Lv",
- "jUmRqikX9BNMc8RjUvnxnUg6B52pUnN5MBjEPPq06OsG/VT2CJaqt9PHAJddTz/iswHXLYYDNxBk5Y34",
- "3D67BN/kQecwuyGy93vOacnnhOlzUOcXotD52fERUvwDsTHtWs6GOrtPcP/+gXwV3GY6n3rgdz32H/fK",
- "7diIYEHEqSOPf/x23SmHI+itgMEQHylMGYkRHmuqd3uIC/v80O0EyxVMBwAp5wO9/M6XL/Ccdsytd0Bh",
- "U3TN7cOcCjzBd1jgefpfxpay7mRz+dUxBiM6Y5GeLC0g1mtfCcY4vDjLTN2i9wRMbHQuoimRynpWbJkq",
- "sNMTGhF7rKzCAJHrWvOnkiyBRxlzpBOa7xBkY3Yc6mz3t/s78HR/ThjWwrWz29/u79oIVSDdAY5nlA00",
- "ycHfE1IT1AePGYQ0psHhb1fZhsN97BxPrOnZzR8CmJg8yQUETsh07p5u6yMAND6L7eiHGop3AATEdVo3",
- "EQRbl8OSJ8QdAYzSdzND3H/nwJz2872em7Tk5ixk1gZlxZe8itnZXpo89Eu3Pm27gGM+XFIjC0EdcLfS",
- "yZMKhEvjZFYF8IpgEU3BrjbxEZpjBqC7s22rAVVC1wKcq6e9qHnwPlqYO6Is0XkQgKwYRhVRHZwknqOC",
- "MDwyJYNiKt0/dZP3rWAiCfglNeWi0aIOHC7ULXwNweMkfwaR/dtLrd7tJFiq21La8Hq4rjQ4MRUkaqB2",
- "gIoLY28EESUjDyzzl54nBML7bsd5vEA4DLe3ncS1yRpxfks9+EMaGySftfExseN4iCMGiR72e4DY0VJj",
- "SrCLFr7EirymM6rqZrFtB3lDmGHPLCDUI1vo4MxkyncuUei2s7zbO5YZB7HptLu80ykXIxrHBM7Ge8NX",
- "y3tcc/4Gs4WFDtzU++1WZXQM5OEpWHAgZDMr4Pf3etulSz9ktiG1klnhiZbJHdg8W5W2815bx1zWvV4i",
- "0r5RAA6nDI24mhY0iNYSF0TMqOpTbnOz2xSW3IXqePFApv6GPrnO5hzyI/RsuSKJhtvb6Pz/Q9SFLySC",
- "4Hjhkm2Z0KHYmCcJwkoJOkoVkf0bdsN+/DF7bXWa8PuDH3+8YTsQnicQzQq76BX4wG/NBYWHuUXDxy7j",
- "2Q0b9tHZOGvuChZAtRbZRZkqND8YEH+9vr5A+9tDtMW4l3OBxM9u2G4Qohx9WwVbdwB4y2DZA1jyxmVo",
- "rBCmSeJQRnPrfJQqE6CjIGcpif9zjBNJbth+H12a6DPhxWwVodL2Xo+Mx1yoLvhLKEsJhNrqqVNB5DOz",
- "B3okdJhtNPqZTPEd5QK2o2cngjBKgFBPs+Cp8OYi7I4Kbnpvkf6k39W2BxG3YOqYRbp/24iqrsuip789",
- "0/NYITBgnPVgIj2aibHTACNJE8JUsrA5QQhK+GRCYrD5iOhJGhM9DDzhkimY9uM00e0dfmzA3nxOsMje",
- "bunfbvMm2XWDVkl6vOtqI5OQxPrNMZLpyF5Le4/B8ul0Y1ifqaiQr6hbHtclaYI0gGAqzNUCbQGJTLFE",
- "jPvhWnb3LrBQFCfoyiwZXUWEYUG5dNtn+RQofLi9g8gdYZpdAzSpjwR8RiqsD+DqwY6goFeBJA2mDhBQ",
- "pnNFmoe4AkJiGFdBvoHx+Gyut7KEiDtZQaPiKCaKRKoMXEbNejzg0wgzXzQQtGX56RkwlN63BEcfUIFt",
- "0VYG2DONHb04w6MV69nI2EybdsxFBZHKVTVYr7a217sBfX3tAn1cDuLgFW+aw2hurew7xM3bGQZyL19f",
- "ZQXv6lTGDKtoCo/RM42Btmimf56tyToZGjPjmyzb8YQvq/49jK7tFkbXEWfjhBpv22NaabrXcHmvn3H8",
- "C1bk3gb/tDPsjvz3o5Yta+y7L92Cn2DwGU4zX4zBl5CQP1TLL8yMijRtpOV+40FoNgL7CBKI5cYRlYgK",
- "QcC3MUpIH9l7U4ngLlfM4EG0vRrN/AdVX8MxgOJLy0ZnwxttZ4yIeysIc2hpZuY0C6s/i9lmFXHnH8zK",
- "N2SPc+YyaFgqFMzGbUQoDLdffLNlzY2RklFmjLZmaaJoz3gYnv2byL295T3ecnXKUxZ/t6dTs+NGrsxz",
- "kdN0VA16OC+JEpTcOUvRmTAtRBUaLYwXDeE4FkTKqtD5haiifbZ59jbZYuv5Gr4/0flfhs5/IcYJk1Fb",
- "E4XrfQo4Y/RpSerThNXD1RM+ZvGA+0dpbf3yVFn3Cjx/1KdBj35KhxE9R0aGJzBBDcUHyNIAtAaq7Hb+",
- "u6dH7BkIenZ99Vnaw/joNCrq8iQGa2fnrWfJ0Nw8z1+GxUKChhNz5M58NYQCKoxHak3y53tkWOObcH5I",
- "4jih3oFaNENDQOVNBsBdIAb0rHMoWFvdAYihtBrNP7lCXl64d+ii7AriWUkQoBlmeEJk1f3ik24fnfi6",
- "D0WY6e0eERRNMZto0yqVxCpIk7DfrUIb0vado7n9eWa9sIcOVGRXYH1HOdgAXwY5uN9cwGjROwsuuBlV",
- "LlhxwVMUc/aDQveYmSw6AGbmJyQshpczuZ/HNoC/fRflG0CPcTJenly8Pjw6QVdKYEUmi2cWYhNwY7LY",
- "ef43ZNtfgQsue1luWmxp/JncAncEXJHXXP9thzjIst4fvn6NolSI3DF58frdFXj99IlO/2R7CzLjd6Qy",
- "AKwz8zoCbhxWPhAyL3V2zsIDgyTMrEPQrOj39w7VWWZg61qEy7SE4LsQKaXMkskGfL9r9OK6E0XfX5Oc",
- "8ntp8/oxnNiwHxMMkR+Y/OUWfIzFIRt9jLlPUSPpxAQRoR46BKrQWAEEHfnEcIB+v+l4Tu2bzvsbZinp",
- "xj0kvunY3azp0vWbvr9hJnZ9ebsimJdknuCIoMMkMVtYA202gh4uALrlRMXRH5oEbjqeVz+wksLXIvDF",
- "T2jLzgxU4E1saR9SaRbtHCOYHs3pagPom5yunnxX3ErDb+NnXXb4cJL639DT+Hc4iRharBAddry3shdx",
- "YEM0TBaVr7CCglfQx2bwzPlYtA+6aC7InbZBtI4uRa4ZCxYrnC8OdMRcENAPsX8HHXA1mpkf9+R/WC5K",
- "VOMAsFExT0z4F3V7mf17mNt+YKKjNsNvl6Rnhtccp5mL8lQmi5ziAlzY1waW4kL3KQZRRHiORzShatHM",
- "ayfsu2U1G4r2xGl/TU4zlNWC0aKsgHGbWCjTOsvM5JWxq3WtQf8j935pE/amrYvceMPfeLkf+dA1mZ07",
- "a4b57BgQEYDZfP/bXG9/l64u/yI5IwHHKmYDQrwy+EzjL7Ux5/mNDHY5riPHNqOFzf0avGvxeGRDOiCv",
- "V15PcX/hO5bv9QIEe/sPyXjqaGxVd2qe5Xq5N9XPBelLcU38tcLbdH4E4d3sKXDCe2UPQZDC/y4H+e/4",
- "lI1XlqeDyNUqWVGyum5OuFrarpOxrvnPi6M8bfjGBG42XVDgZqA/ydxNyFyfMjKq8CnSNdisAL4iyk2f",
- "g7Rc9F7VE+sGhHCOCtVCCNvIMveGMFtWv4Vc3gtlgnBo0aM+ieX1c8RVkSOamaAsmbPUjssFM2TbtO+f",
- "8qx3o4WXwyIomQtJ9B5HNBfz9jXYw9ky9MKeCHKNr7NUITdiqeKGI9Bsj9YlpYNejnem/D52+bshlYnN",
- "3mwuVUvvumZE4RgrHDCZ9UjHeTrUenkNwZtzLNRgzMWsB8NBMeSIx5RNTHlo4/C0va5t2iOP6k1CFfJx",
- "nkDOAHhA0u1ItYDn1npgsy0ZU5TrTyc1Ga4cClJYkJ9+YUQZhnjhQibOF7sv9nZeDvca87k0JLWOUqmK",
- "2VvytBOOSqrpY/a2XxUe8PZ/WpptDJZcTTZWlQBm5QYRI719Dzp1ZIUQzHAkfpIh6zprWJ6FDW2UGHUq",
- "bTByNtvSE4dTa9DD7qXMTKC8mlkmwOo13M96CLI0Y8Ab817eK09h6m1DfQeVirrw/QRoJPiKerjud/lv",
- "q7DJD3ReAxkfjyWpAW17LfldNvkKAbbNEN2yt98FInli9/WZDFXu+/Ymw7/OLhAW0ZTeFcplGO2eEHRx",
- "fFo2c+BdpFlLXvJniSkB5Pe19oSFs9mk+ETnX2NSeHPUWxWQK9PoQ4RlEYvtbY1lqt4N2Ebb12xjYfda",
- "WADDTcibptvanz2WgFofc5eAtEBaTyJobRZHDWdjWSLiBxskg8/wn1t7z1T3VvEIs4gkmnl8sWjy0VLp",
- "gmyTRZOMMWNUZcwyn02B6iIYpBwt8FcMFvg7XP2bLS1RxTL1uMT+Nc4/vcHw1LBSY9rzyJdmrbjgl9Da",
- "Ruy1htjOAiFbH+cT5X47x31BlGWpszZi2nWXtgXiACuwLLBNjabVY9FaGJZnjCoKd7b6ZGkmciK86jq9",
- "FnQyIeLEwLOZ6wEzuJ2p7n7AAmrtI5Ps6FFDavK6fAEIzRKQMmt4uv3dCP9aCjGxBpYeHeeaDai3gLya",
- "ynW6CDLw4CQvjDwiCWcTeLjDs4NgHxUqHmRVl0leivkHecNCBZjzettdPQ+/z4MlZJuCyuHyzZCQCGo4",
- "Y5OUyKv2fMN+K1fHR35x/K4PthnJPYOq1oq9YVul2sf2wdmIJGgkCP4Q83v2zCapwHEsEb8jQlMzkhzJ",
- "KU8T+4xsRG5YKkvF+yGKnBESk9jUiq66uIx4O7V7ucTBBUuHxBINizLpHj0kVNboKss+69+wY+NOgt0y",
- "uZTgIEQEmCss0oSBJWfSFrsOOKnsvGEvlT0LP2r2C69seYPLyS9cXqwx9WTKfCtvVU0xeScSzcauxUlV",
- "lqbWcGp3YwveKGvYt4qiuYK2jxRCY+KGzIzNwbJ2CU/RNGs3yi05eISymSjGnIxbGALZlYw2CZyWtgkJ",
- "zTv9foOOkpsnWj1Zk+fOiW4Hu7Bc+WScbkwie6SyJDYx88i1jPZWXogUFEnNzBQrUmnszBxr3UD8gN7q",
- "KcLyhhll0c0uwLvGqpHGQHWBB/bpO6QzUeRjoXCRqfcZMs08t8vqRplcNmOWBdTRetiw0p1tddnvxrZy",
- "WDlhgkZTTb1VNs0u8P/K6Yf+Lu6huLQZOXet10vk1/Z778sDOficV4z9MjA82iaCIEksQxsHPRwIR4Sw",
- "cB3lLoJk6KZO2IxDxcIIjqhUSNVv4vDXBqSv5JrinV6+zFaF2/xK0uXS+6GS40sv55yqNHA8ceC3jKzO",
- "CblYQ89jQNj/Buarr5FsQrRBfcyxmubao1CmuT4v5Wplq+udsFDGHgJ/mgrUm9O2wNEHrdM95wUkValy",
- "qR518drWmN6Ep7ZSgr/GWZuvRUP0qG7agmwIcHqhtvxf/vXx34HrgWqzsv/lcv81PK91pnECN1vQNr0f",
- "uKxsah2TkUgb0c6PHHg5b9zJyj0G25B9aKZpuD609wpTLI0y12NAbtanY9s6E9LZy6VMrNbdKqxq5pn+",
- "mZEHGbJ6+QGnrQetaKCRJPYPSUUtGX6WZigfKvicZD07D9ObpWPXxhTn5hzeRTQ0uVBcNq4y0p90xbf0",
- "FtZsiu/6Lm7xqiV5KjwGjzOkX+bZZMYTeBEu+FyXqKLKgpsw0yoUbkknbKxVVmsfnjyezbYCR56Wgf3b",
- "VGj4W8SG+TkvHsKfQS05mFKpuFi0dIPYSo/gJy3DUObTB/hBSrD/amH7GyvToq/Gobe1t6aEMFeWdy2O",
- "m4o4sKTyJAG+oYKuKBRLMm5zHi4I7vKSzu1zOLjZA+KgjfFcAvHnhSPg75jjuyFgHB7sgy/zzgvQVQPb",
- "XbbQFoBRpp7v+Y+qXg2Hu7svhtu7z1/u77148Xx53djvxOg/DRFJ8RWeVzX8Scp8L1JmtEA5xbYTMHlQ",
- "YKtjgYkQ0+TAxQQz+omySf5Co9bmh26bMvVh8K9ITjf2oXskc99MGWI9g+GntHSPZaK7za+LGfO4ZPDZ",
- "/OPMeFuX6r4s2DV84eIGW/d1S3OaMi+sVoMDJb6rfHsJERKPwLdmojq+9WSOxd3j5S1fyqMCQH86d38X",
- "TG3oyJfmqzF0Hn7Q8oydP0u8p2pKmW9vu0ijPBrJxRzl8eKyf8PO85ij1qFAWbR2ZrLfMD9ibzJVRPTu",
- "if4HMtIDqSlm6JeTa1QMukJb5KOd1RY3oLEpdcBn81S5ornyWU3Ek8Hssfd2d+2BT5XVfmehT0XHQIGG",
- "WnkGsigY5sRN2SfwVdEcOZFSZqnySUJ9B8FVsB8tZFX3+zQyakSoe9ixXIB6gRxzwSdQIcp/jhLEU79e",
- "Br3JnpRs2BxwE9VbBXYZT2z2LQ/G1mSsvHX6a7LalJu8Xo3R+lqT/nr95jWaEZaiOZ6Yl1l+hXlTYyFc",
- "h/RXPiMXeNIi1kOr2sFUzZIi7xShAkhsp8x7pNdhINMw+klIYLhSCpK9l/svCpnH/rf/438Eco9VGPHX",
- "bBoNxdOheW1sFdzDrPJBqqZFI9vU8FOLVqRbpdMfpC2RlhU+g6K7WUk337yGSgn6gzT3XhOBmTJXWmA9",
- "6v59dMgWgXlQhBmKtEmvplQiwuI5p6baXUxlxO8IUC8ViN8zA0mQg87ccjeohNwcS4uL2KJELC6g5en9",
- "y9eU2l9PvJsetOEZ/anJAuJdTQSJ/4EBI6bGjiWicGmuChub0+rgM/z3LZ6RrzkrlyLzFYaqihBc4p2f",
- "YarGEHz5sxfl+x2c364sujd1fLP1YKjMcfPEsusLty8hukyKXxV4b4J6bcbVgNmY8VWj3VgwjbYLZtHv",
- "h71/4d6n7d6r2/c/hSyk4F1o9gAZXvLTRBFRSMMW8rJExSTpYUjbFOVxlm3CJ7T+LruYpwXaIngoMFog",
- "QWIqSKRsbgpXn/js+MI21DZK4HkqTFgSGrtGvpclmJnAHz0fuch/r7l9DFD1ul++dlmOg4OEDgaCFk3h",
- "4fbey8KGT5Way4PBoP9jMBXvkzRolAaaHSKzF1YoaBphcBpzFfTngpskxqUXrrB1dj89xdKsP3WnnjYw",
- "Rzj6UEvsv2IWJ5bUz/UoQ+T6oDgVzsq11GOTFaFLEhHqgqQdim2JPB4TYzX4zKF1PvloKuZKRO3hjH8g",
- "TNZwy5GDfImsO2yevU6k8Li14KswQi75bm56bYUfPGnwqo1rBBxdXZ5qpCri4vRCsEoVqoxTK6WHeysD",
- "+35VyeQRYbXCdGKF3SqSysnV+tEeLrL+3zqZ9e8kfQoCpczo7vC5TKDwtL5YxGs+MeIAjH2emvfxBnE4",
- "K8IuCZwJzRkgrEvNPBlMQfGgAfkKbarHB2cCFwVSzrwMX0e9iQPwiWY3lPLBbCHI+VV0ohGsq2SF1z0o",
- "kWiEJYkRZ85kjQRVRFAczkTyT9Nr02VO3DSBo9xheQVPR7c1Zhv5M9tfR2t6KxarvnYBcly95uo/rXmw",
- "iegYGPsrgtr+9GD7DrJDwnKeotoeK6rN7X4NV2Qi+AGVVg2vNBdazRljQ0LXTFBLZ08JyjZQ7i/b+EKN",
- "v6q8Xel5MHRfvb6qJ7BblFfduJxuLq5qyqEieDWZp4+MiQShDUvpP6zykaH2p3Krj1BudSWJOlBEPiB1",
- "dYEZgmbLyUcSpcZu0VMgkTKTC63OgslHDyS2JlJtnDX0JEte3xrYzXpWzG29vQlg669Wrw2MMk2e6hSu",
- "K6O1Ruly9oLxxF3Y83gheJyasFHTqNPtpCKBuBLjKY959GnRj/hscLcD7GXnqdbKswyireoEpKo2qrN8",
- "htYRWExnWHUx1gzjKoH6I5Wrg7YdzL+qsWNVkrKtOlaWThfyIZr3Pe4sb+coRk2tPIFNa6XHz7I+uexO",
- "+SSFO7bVF1F+8OxBH34K1XaK/LRpxytQadtRTKYVb5RiipW2w3iBLIUd8p0tbccCT+EMMzwB0WtSO8Qz",
- "yqhUojy+/j2b4P2X/xMAAP//akDiTcVdAQA=",
+ "H4sIAAAAAAAC/+y9C3PbuLIg/FdQ+u5X48zoYct2Ht66tdfjx4zv5uG1nZ29Z+zrgkhIwoQCdADQjpLK",
+ "f99CAyBBEqQoR3KSOT51qiYW8Wg0+oVGo/tzJ+KzOWeEKdk5+NyZEhwTAf+8wIq8pjOq9B8xkZGgc0U5",
+ "6xzAJ5Tob4iyMRczrD8gypD5A1134Ov/uKcs5vf/ruiM9My/rzudbod8xLN5QjoHnZ3tbddoZ7vT7cho",
+ "SmZYzxhs81y3meGPrwmbqGnnYHfY7cyxUkRosP77z+3eq5tfXGPz1791uh21mOuBpBKUTTpfvnzRvQSe",
+ "EWXX+itW0fTsuLrSqylBI/0RpfOE4xidHfc73Q7V3+ZYTTvdDsMzPTi0uqVxp9sR5J8pFSTuHCiREn9R",
+ "/ybIuHPQ+f8GOdYH5qscOBg0dEcJJUzVARTB13pQvgKIbGINxTGP0lkDHLH9vhFIvMk1LCcf51zUQkLg",
+ "60bgyCbWULyXRJzMME2qYLy/eN0jLOIxiVEqiUBEt0M4jgWRsgYuaNMImmEor2mB+reDtC2InHMmiSXt",
+ "+DesyD1e6L8izhRhwNJ4Pk9oBJw7+EvqNXxuixIhuHhDpMQTYmYsouLko2ZInCBJxB2NCCK6g0ZBnXgJ",
+ "zWbbDvKGMNURZ+OERurRVnNBJE9FRBBOBMHxApGPVCqJuEBSaUEYWYjWtMBTLkY0jgl7tBWeMZmOxzQC",
+ "qTInYkalpJxJpLj+U5MgUlMqEY50jzWt84wZKgHoHnGtHm1qLl0jaZ6xO5zQ+IL8MyVSPeKSYFokzLxo",
+ "xOPFmlb0lqtTnrL48ZmNcYXGeuo1reSK8zeYLezeyMdcEMhilM45Q4pzNMNs4fZKrmF13c4FUWLROxwr",
+ "Iqp66W06GxGB+BhJEnEWSzQiYy4IUmJB2QThCabA0pnBtaPVSkADUaZ2h0YD0Vk66xy8fL63vd3tzCgz",
+ "f+faiDJFJkRohGityXCqplzQTyR+fMTfTwlDqQfCWijqi0MRjHEYzyjTxsEhSEg3d8BydlCNubBmAsOj",
+ "hAxiKvV/rYiV6J6qKZJpFBEpQc+kEmEWI21KS4Vn8063Mxd8ToSiRtGbntUpDUhOkBNtCRGmt+vPjp0U",
+ "foF/3Pi2ef61ZGN0OxGfMKr4rUxHf5FIabO3Mu+RaYNsG0RjwhQdUyJg8WrqFoucvVI4GeCd0TDajfd6",
+ "ZH/8vPfi5avtHh5FcY+Md4a7e/vP9S9Fc2i4/7xwGuiHbP+uNaQq4IJZp9lEAwY7gyOlbbk5ZwXA/uJT",
+ "1o85+Q/7Uz/is053ZUOt27G7WwXljylRU1JAEbQlsdk9B4qxFe24I84TgpkeOKeRqrHsPrmVWqrxFzjc",
+ "Hu73drZ7O8+vdvYOhvsHu7v/8BcYY0V6eo7iIvcDxqhv2f6ZLbib4cjOfpP15EArehEZSx0JglWQlXJ1",
+ "BxQV6YYg0hAj92YPgY1gti7SNncXmEhwzWlS0gmbwcm3zEs1NPK+YtejrVSSGGGJHLnrafVMz9ZCNP7h",
+ "Fvc+Hfb+sd171b/9/3+5vu7d/PIf3m/ww/V13/5083nY/RKk/zEVUt2aA0hogT9JNKF3hAG+ALE4injK",
+ "lEVwiVr+k09ZEfAdqxSyv9twZYKXATXGM5osWkJ1zMkagNJ0EuDP11QqzT9ARh/IAgxlQ06IMnSuLWjV",
+ "pxxtkf6k30UynRNxizVBd4E+3L9xGlPFRdee6G/1t2d9dKGnRbNUKnPW0IMueCq8kQm7o4ID9fZRZgBC",
+ "v7EmT0kTwlSyQD0UTUn0wXy7NUCSWI/opG5B9//ZyeHT/GkA1AxKFZkBLspIXY7GGf54Znrvm02wf+1k",
+ "bbEQeFGRF44pPIr1CcXtTwvp0UIhY20++TJEI6KLKIuSNNa/lPXZ2THIErlgkVXQFTHSSk0CgMZn8JMn",
+ "Rhj9Z0oCyvNxlGRkoLrFqkmHgGmV6ct7LJHtp+lL/y4XUpFZk3bZPdjbf6B26bYX05tR4MZoipsVeOTt",
+ "L6LS2nuAIbvXbVR6k9AuUVAuvNvI6a+SzKWZPQndQhgvnxlcEepWc1gTljPis/bFOE2ShU+Judh0hwDj",
+ "rzbU2UdnYzTGiSRda5Eb/06+QWgrwgz6aiaMsCLP0ChVcEYtyvsploiVphl4HpVnffQGsxQnRmxwgYQ+",
+ "u6EZXqARQU749duQRFGiByQbaAM1xQrdE0GK2PEVQQa/Bg7gwFruSGKUnDGziFMvdIwk1yoY04TEfXTE",
+ "Z3MsiLG0yo1BrmvtGBOlxVhuc0H/VBAJ6GdpkmhskNlcLboahaY/AJ7BugXbY3FcxOo7lizQXBCph6Zj",
+ "VCAdzXYai4+t6YrKrduxWqL2pATLy1RJzj+n7y6OTm6Pfj98+9vJ7fnh5eUf7y6Oq+JxGXgl7RpQTrlh",
+ "niu0Ag82qtpjkpBWqjY7+8a6h+ZEc9o1p9yx4DNEcDS1/Im2HH60us2I9dnDtG39odQeiAAmK9MeR9fa",
+ "GW/1wqsQHzsc1aCnDg0t5KVbKgy5gjIyNLHqBNnOLZ8iTKkZNQbJsM2xvnZvH/dMn5H9xk71boYmy+vF",
+ "wc7+ps71BZJeIjQUpklQKGqwlbUt/Wtmn1yLMsGz1q3+8T1mD7PP37e0yw1YnpGQWxjfr8nuztCe6R4m",
+ "mf2r7VcHO/sHu9vrNtZPCu6Uov8NgHtk2/0oFUIbEc5Ot2LXB2x9pnvV36LtQUruGqTyRiz5gI+lBSQP",
+ "Nexr/CrGYg2apkXLzRltd5Tck29ks1kyKdhujp0CNtzRu7enZxdvTh5gt3U76Txuwdp8jPQOI9O6SfRv",
+ "H+xsr0f0tzMiHbM1aoLXVAaWd44nlMFBLrFeN41pez+iuMIJMjjXon5uGmvJOyMKx1jhisyfYnk744LU",
+ "q2j9VQ8F5xeC8B2mibsmWWoa4QmpJxb9FTG4kCtcuHUbLtl2trcLl2w71Us2M+2tpJ9I0/2fQdycCIDD",
+ "B0BLyEYIls0PGxGgTdgfVoJghlU01Tp6TBNVwsTeiyZIhjt7L/Ze7j7XrZpuHbtwtmvw3BpA4NLA25qO",
+ "J0earh4rxktBigC6Gv2aBjiHNUs0/iY28sl7w97L70NADmg8G9mklKCjVBGJtkAl2fsQEBnFqxFZPVs1",
+ "KjEjncq3Bk5dlnQXw2qKN31jYCEqXxkEQTrmpHc5oxCWtaELg8yWTdreHPTQxcn568Ojk0uEk8S4pHKz",
+ "dotxpU0mqugdedZHV1z/hTB8dj5rAj0dfZuO8ySVgAVG7hFnBLoKMuN3JOvNIaxTGa0PV4SaiMO3El9z",
+ "IeGuH2wjLtCYQmCOwor00TsNBkQdjSlJYqStakES8CYlBN8RO3rKoilmk5LT7JvfY1T5V3861UvRvao0",
+ "cknZJNGLutcEYhxnZuVajaGtnZ2h+RtwNdPEwBm0r3IrxoIockwEvSPxIdAJZxdYkSNAlTYZEspICQvD",
+ "7ZChXBjrKMF0drWYk6M4hMLmzqeEXEZTEqfJA2b2Ov8fIqSNdFgNgN8ET+dEhPu3gOE1H63e6S1R91x8",
+ "WL3jueBxauIcV+44EXj2oI53V9Bi1Z4XhM5Gb4ia8rhdZ0uTx0RG1X1s6vEGfzwlRNPyGYsK4UkxT42R",
+ "lpkLr7L/9V+9CloM1hYrjn8eqdbj9/Xoy4ZO1VRrb6VxG1jxfnDBI5okK3Ba3rz1DBEWd4SnquUMrvmZ",
+ "CQes2r6RFg7ZQtuOqjtdwkHhIT3aL3au9s4Fj9pOkjVfeYY3fLUp3vD2c8RkjNOkfgtiiictV2iatp9Z",
+ "Ti3frZHpYjmtX4qcWkZcDxeSj1M6ourcHtCW4se2v6Kqoq7CGBrjiCZULQ7jv1KptCF0RcSsVdfJjKwf",
+ "uZMZqUXuZEbWitwJ+A7Fu3H9jEb3HiYQfq3t1tfkjiTvxkdYtFM4doRfsVw7pszILRkna92ad2yP33G0",
+ "DD1r3hQY84JoK9jclC6b/4LKD+/Gb7hQWJPyZTqKEizlKni5JHdEULV4SJ9l4F0JzOSYiKXtqkZMeH66",
+ "CcajDYxH18x4CZGyke8SPrKG5AVJwDUmp3TeiuFMV21KrtyVxW0VLbRszUlM4fn6N0yPWotA/XGtW8ZT",
+ "lVAiTj5GSaqZsiWiqt1aY812PaVCqmOeJLiegbKmH/VxR8q3I3GMF/JqKoic8iRuifV+WyxkE606wwpz",
+ "vHF9NzTwKUS9sGixbuRsguxa2yRzrChh6nBC3uCPlR7NHShr1SHBEXk3vjSPBFuyQblTayaYC353OScR",
+ "xYlatJ2s2Gelua7wR874bJWp8i6tZxKYTYhl0lbzCKzIiYxwghUXK0zj9aoVHoVWb/BHe4o+N06Mryfh",
+ "wvhhd9Za2E8QOhsdcXZnHEWnOFLmhWK70duNfzIek0jbwccl7Ji7u2XMA2OsXxXCsGuVO2ZEwe/eWlf9",
+ "cmLLesxpxdpoan9FWev2reVg1treLD50u+4IS9sKuax1awa1L61br0oqPtdq9xjTZGFZdY1k5IZf1cip",
+ "9GuPANuzjZGTt92g8eEmWb/1URp5/eaHm8DKgXfs5GNEpDyaYjGx14FfLRbcHBclBbbWBbRmByXoZGJ8",
+ "C5ugBTv8EZ5nox/O1BonSKP1q4K03luRRmtVESmj6t34DcEyFW3O7aHrtfYZXGjxprCzvfNq/+V2PO6R",
+ "8fP93ovnL573XsbkVe/V3t7uPn61u7v7AvuxMmkKQS5+hNnzAEkBQJc10UMaqDywDBcA9N7nzgXXjKdH",
+ "7EKiHoiq1MBA8L3+EbOIJMXYmhIM72FUL84SJ8m7cefgzxbpaEzfy3Q2w2LR+dL9XA4LADhuxzQhDM+a",
+ "3splTcyLBNMRFRboXd7GPNrtz+Nxp6v/ufMC/l1zebu3/er5are3NqRm2Z3tTbcSiq0M2H40Kh7xVJW2",
+ "0ItDtQvNMVTclnDQk0OaP6a5Isf2qWDlyheamn+2imEJbm9zFIsNWLk1UZlLg30sRDbwgsoc8jzuaH8d",
+ "0T6l6BqHiSK4N3Uyw+CgxasFQVQqGIlNCG15wyXCUUTmEHXCRZGwAztlQ41b5YPygxCXdrASJ+t0m4pw",
+ "siKkuI3EMAsJxCwOzI4NDg8PBy7X0wBaD1YWmoL2BBkTQVhUtpd3hsuiDL3cWhmY3vKW7K2j72rIhfmw",
+ "lJ3XsYX2he/yPnkGLk/kByM//3Ch3AVKzPoUabBNqCdLExPoWAxvbBdiHoamGFj+oLhxq2McAcqmEMes",
+ "UQstM1xPpCE1sU63+ks7IBlnvfPjU9AJ0iS5QZQhLKIpvSOtA0Pbg8gFnVCGk0xPV6F7Z5tkiioLeId9",
+ "JDH6x9l5CMROtuT+JzovbmdbzWz3Z/VNzjoiydEYF+L69ta0v3PBJ4JIeTsnIiIhxXeekRdyjZFtXIqz",
+ "bYZpZ1kynQcqAqMGG1Bb1tw5gqnH0gXC3N7elOY2Yj4XlyHy9dRAeXFhagpIkVrODWx5QfKFtI0R2keY",
+ "XS5M1FApQ5fhJZu3UdsKScLvIc0AuADhNbDshALK7cg1OVD0OSJXTV4WFJOlrai1VtM9Tk4sbw+OvfJe",
+ "ws7BGPX4WpLfkrg0grT4KOnw8LB6/louZdycv/J40TivOSA+HHdVTNSjQFv6LWzPiDOFKTOJKVwQMYT3",
+ "wiiyCrD9UH+2sA2KmRr6bUPgj3lkFtDiZGWZ9KjdscFb0doFjENKAaL6zXkbVJUepVQyHRweClL2B7d8",
+ "vmemrHMYwC3UmBITve27Dgwofc9rcPb29vK/3h51up23767gn/DqyP1x9va3oLPAByDMI4du3XZ+3242",
+ "uKvSYTuNVVh85Xmp+bl+n+peRJSkI5iENtOCfVEV4BvM4LF7O4idxH+YvKwuh8d+dHMg4y6RGiso4jFB",
+ "9nqqzCnP9wouuOFwd/fFcHv3+cv9vRcvnrd4THTEk4S4K6/yQwb7Cc14TJJ+MAndHbm9yxfRhA+31oee",
+ "jxKsiFQPmM0i4DbSapaItiP4++ONoshH9aAhguLJGD11ENbMWsFFt7wVQfZx2wnvEurFHU/VPFX2OYIm",
+ "9P7DRJyb7pIEX/ot4MhYeLqEUdZpjcT2/Wx/ZUtyxbpMnJWk/5rE2DcyErs5xDdhrIB1fsIEjaahZ9vu",
+ "S56UPDYeb/NS1IX346T+ZeiDRJA+k1zST+TXhQq5v09pQpCkn4i2s0YLeH0HyW40lSdkgqNFftx61ijJ",
+ "94av9l49fzF8tb/slPiAY76DIXSAr3pkeBITcRaHeBgem5kG6Oy4azL70PxMqU8/Nm8Stu2aL1eCPqEp",
+ "llfko7ogERcNmaEw0rypDxLC5tHcso/Jsl+eIQGDuKxPmbfaw0j1VDbFcrqMSn7XbVpxlJ/qXgvyEQkl",
+ "xDhMEmS+IciaS2KkeAXWVtb7az2MRV7Ffq+a7+4Afo7VtIGm5lhNtYC6o7Hmw1TA21PjB9yis1mq8CiB",
+ "vJjLSUwVdrdpLaC6TrL9zA5TQVnja1jYwzIpZehvkkJL6yHUXjGOdscvey929172XuHRTm+H7O+Pdsc7",
+ "L0n84gFXjBk87NQwUjV5kINIC0GP5WxeVCcznUhEWwmdTJXJ0DvDqovIR3hK6i5vbmkMr4TNQ8TQM+En",
+ "aVgvDR9T0D1JtX8dqfZwSVZ7NVYQZ9Lek+W83g+l1dgQ7dSuOLSwQlb4QC617C/N6lD7wRfUnzsz17Wz",
+ "4Kl57J69pkfu45fy4md1E1pIkPl1pEkHHME4LeXdzaeb0DuCZgRNOC8EGjzgxFXCnIMyiDeoclObjMs/",
+ "m9laOzamqZBGeA32tDlq3ibcZNAKU6b7qtXI/ZRGU8CqBewTnYNAR/c0SdCI5GUkcmzL3YPBAA8OB6bP",
+ "YLg93N8e7uwOtFjuR/KuhO3tvZcFdEP//i/6/3YEKDb1+eWXQf+X62s9QvAE3M4bZjajxhvmmzANnrFW",
+ "9ZJq7BTS24v3d3ovdnaj3quXhPT2nu+Rl7svd3bGO7sPsFMK6wkfa7mUdJRkgEEOCOk7Nf3oJ8pu3RVN",
+ "Hgt1UzvxlaHUZqq25Aw+B2bB6AfsG5JAuJMKppc5hQ/GRRvxOUEjLEmMOLOuE+tHucNJalbXSq+BLjBD",
+ "F/UaCIGyWqNsQqSG5yFgZp1RXh+tigXC4tt6jysklNGftYjNBuy3uP7fHYZZRqja6SA9iFjjhFWfTMhL",
+ "U1HPWcWfcE4exW0UhK1NUD6F2gSoJgWINrC9/B/VtIU4SyUSMqigJ0TcleYAYkNbOEmMppniO7hZklQq",
+ "49UXaaRSAcZMu0RIxZwmba6C7EXqr4uGVJ2QryjLxG5Eu1tFQV7phl+dCdCZOWcrWrRmt/JtaOp56bct",
+ "S3QPgNKo3cJW+8i7aUOUddeKp0Hqsx7Mks2xPsrbPFEdqpoU5RoGv3DOJnKirkDa91OepSeHk873SN6h",
+ "XKgVwoEz7TpDqYOIfTindTt3dddp1gvvbp63dnqgBZ+1jr8qBbbvLL19hrUWmD2/rGnD9odqVRFQe5dY",
+ "2UjH/O6op02iKZWKi0XVrv/BuS3H+hOrlZfwg7DLihxhjecaQ9jcM0Y8SWcskCKamZujFaz0o6wPuB/1",
+ "kaHVRVn1NlbvCBwWwqa2+abJfJwmxm9GbFbNwuli9avaVdJZeivsevjKQF+yLUc+hhuOhsaqGGdbls1U",
+ "OCcmRMpbm2HSZqVxf0YJlyS+1fQm7iDxJp8T5v+dkLG6rTYTdDIN/W49Q8AC5l+hQ2idb978XsxszgXi",
+ "YoIZ/aRXWIpJrLhWzlYLJv9hxHZeUagsurMLwzVL7qAE9tz2G5e7M6IEjVZ4nmWge2O6nTF4/159seSF",
+ "JtsZ8qsAg0tXOoXP0wTwDK9bbON/VyIl14xKJG38WVbZyVZ66aO3ULpFTYm4p5L0rzWnFR8SmGrrGb0u",
+ "vSNZCdmj5che+qxhHrwBsPsP/v8tODCDK8IclgZQpo1ngE+xNC2vO4PrTtGrlr2dkYPh9nCvkma2mHt0",
+ "0P95uSsXlmlLiWeyYCW1CGAvK61YcVuYxT6GmFrVriMbkw31tPtubkNK5l9JxA+RGA8k2veSGBqFXRQ5",
+ "CW+cZmvJtZFEmx9IGtCXPI20jZaO0kcnOJq6XbTKXSKqJESyaPi7Ns28oYj+NdMyoJeQO5JksIBbzTVB",
+ "kgAbaRnUR0c25jrCTNPwNcuaKQ43tsz44Dx6RlNKBBbRdDFQgpDcSWckbTsHsuGMVm9gC4adRVz97rzJ",
+ "1VYrxYOL2PWlh0njX5UhowXc64JmzIK4zksb3Jg8f/mzmpqgCQMRGi3MDXXxpvAYy+mIYxHfXhAcL6yh",
+ "fAbuZ42+Hc21744ubs/dexSorG02wYffN7Bz/NZHFbw3FWgC5ZQDEnBDwgfC548f+KKoCibg6it3MUi7",
+ "IGZKwHYzklpK19auqr7ngd+9d4VhIt/KomccNEhSFhFE1U8SmUsWwtSzJ5o3uP/+qeoBpHRRE3zk2VjC",
+ "BB/BvaPN85+tpkgYYaX/NjfN1q34ewa2eG0GQAhRv9vYkYYQFBvi0nQnX+PJOItrw0EO3Y0b3PrXvPlq",
+ "I3z1H2rRX/lqvN1LsbPYjL/aOzF3iP5J+rXQpoQKv3RmZdEtSl57JZqsyykvtR6aoJWNki2SJ2RVB1R9",
+ "GejCqIH99wrBh1ajTb8mZBWGqz0dxMWoIzg982I5oY6Wpz2uT+IYaulpWO4oufdknN4/E5cgy9w43AsQ",
+ "/gdSE9flio8M6opK22IZxUmGLyvV6Ld7r257N2G2Cwu939MZZj1BcKwP5AaS6iO60Pwt/Zb+btUSsdcI",
+ "TQRm+gQ5WhivSHln8uvZAw2255I+SPTRRDfVe2K+1tUXqUfewTJE+kVHljGC3vLsQYWPihBvgOI6nM8T",
+ "Whdq5akoPJ9r2kT3XHwYJ/zeWAfwxY/TLN0Wm4DO5Ud5GN0JrYrdsZ5jfOIsqhJNwDpA/27hZD7FLJ0R",
+ "QSNgt5Tp80/EBUGaNwuXHSU7p7EM3p+HvX/Ybb5pEajnMJDjr3b/6gKDze/mjZDZKi+6dvmWHS6vIWlG",
+ "vcdZ3O7GXMZtyKjgMs6Wqhl6Q+S0zrs+Q4M2YvtxvEc13PBHkb2/L7aoXghWGaXrUXCQaahUTScMz+bM",
+ "nsbnb+zaPo0vxFYvD9m7LMU3hApD9eAyzasA5XFx4YHTzsEOEl6+9mXVoWaExXqQt2mgKFX21Z26YtNL",
+ "q8tDbN+6N975LjuGFoAp5cQtvz1msEpEXCsUqnO519ve6W3vlIVNGznjg1LJ+FoDjMrboZqym8Pe7oPA",
+ "SRW/IIzc48QmYiztTaq4Pp7pBoiyWKtxsJqqDzu8oVxKzIax9JJk+bAVrsRTnyXhrfecJvPLLq+hY9Ga",
+ "VV0pqyCV5KPatm3GHdOEvF3+Bqhqi7pJbjVl2YyErWbTI9l8xKU14I/ozBreNordpjaCgntejptleYzn",
+ "eEHEcuxDszZgQ8NLFY60vee9hGjJbWKzTQYCyHRnh/dK3ZbSswwrQv/m8/BLXSaoO6jSphdwmiYBFaV/",
+ "Re6xln2XIxAU/mi1yGwGk1y64rb2xkNvz8/a7IMbMrjfpSGvzt62HPKdmprtbYEHuH/NsNEWD2aGEB7e",
+ "FcZbAQ/QMYiH0pDt8aCbr06VOSpywnz7Xw8kzFAIeKs8HTUpOh6czEaSKBVULS61rVGoYX+YhjyDh8Vy",
+ "9Cn4qg9n+BPPSpijs+PzPjrTKDJWz8Xp0csXw330n39coZE+/s3BtohsfikHAqwp4ffGpEjVlAv6CaY5",
+ "4jGp/PheJJ2DzlSpuTwYDGIefVr0dYN+KnsES9Xb6WOAy65HG+YDrlsMB24gSPwV8bl9hWDqyXcOM4eJ",
+ "dXc5G57PCdPmduc3otC7s+MjpPgHYkO8tJwNdXafwB39gXwV3GY6n3rgdz32X/fK7diIYEHEqSOP//zj",
+ "qlP2zuutgMEQHylMGYkRHmuqd3uIC/v80O0ECxZMBwAp5wO9/M6XL/C6ZMxtWJzCpq6D24c5FXiC77DA",
+ "8/Q/jC1lT1fGF9QxBiM6Y5GeLC0g1mtfuZs4PD/LTN08Tkqrtv+dErFA70Q0JVIJ881mwoegsIRGxDpM",
+ "qzBAIJfW/KkkS+BRxhzphOY7BNmYxU12tvvb/R14yTYnDGvh2tntb/d3bcAGkO4A6tIOsiLVE1Jzxy1d",
+ "pWptGhz+cZltOLgn83Ln3TwuzlxRSy7gHkGmc/eSSR8BoLE+ssLoWUlp8GxmD34g9qhShd0VLrdK380M",
+ "YXCdg84/NWryvbYlrc2ZyKwNKhd+fbHzL936zJCCyDRRhRrndcCZQttBCPe/rhp6FcBLgkU0BbvaXBdo",
+ "jhmYqtRu22pAldC1AOfqr0Br3n+NFsZlkuVSDAKQ5dutIqqDk8QLvXRV/rudmEr3T93kphVMJAHnlKZc",
+ "NFrUgcOFuoWvIXic5M8gsn972Rv9uuFt4LrU4MRUkKiB2gEqLoy9EUSUjDywzF96nhAIN92Oe1MLwmG4",
+ "ve0krk0LhHOn7eAvaWyQfNZWleshrAYkekOBfC01pgS74JkLrMhrOqOqbhbbdpA3hBn2zAJCPbKFDmzJ",
+ "cPeSDrrtLO/2nmXGQWw67S7vdMrFiMYxgbPx3vDV8h5XnL/BbGGhg4cl++1WZXQMPEsvWHAgZDMr4M8b",
+ "ve3SvcY325BayazwRMvkDmyeLXzVudHWMZd1wbxE2pA94HDK0IiraUGDaC2Rl2836R9tsiTubq686zGT",
+ "4lefXGdzDs8FezYjukTD7W307n8h6rz5iSA4XrjcE+YmLTbmSYKwUoKOUkVk/5pds59/zoKPTxN+f/Dz",
+ "z9dsB26rBaJZ7mi9Ah/4rbmg8E6laPjYZTy7ZsM+OhtnzV1OVEgILbsoU4XmBwPi71dX52h/e4i2GPee",
+ "IJL42TXbDUKUo2+rYOsOAG8ZLHsAS964DI0VwtTV+QfHowN9lCpzX6UgOxaJ/32ME0mu2b6ryI+Fd4VZ",
+ "hErbez0yHnOhuuAvoSwlEHmip04Fkc/MHuiR0GG20ehXMsV3lAvYjl6w9P+CpyJc/x9tkf6k39W2hyvB",
+ "bxbp/m0vGLsuqYz+9kzPY4XAgHHWg4n0aObKWQOMJE0IU8nCPpElKOGTCYnB5iOiJ2lM9DAQ0SxTMO3H",
+ "aaLbO/zY++v5nGCRhTLr327zJu4yHXyteryraiPzPtcmk8RIpiNJVHa7a8REPp1uDOszSVvzFXXL47qc",
+ "BZAVB0yFuVqgLSCRKZaIcf/20u7eORaK4gRdmiWjy4gwLCiXbvssnwKFD7d3ELkjTLNrgCb1kYDPSIX1",
+ "AVw92BHUDCiQpMHUAQLKdK5I8y5FwA0R4yrINzAen831VpYQcScraFQcxUSRSJWBy6hZjwd8GmHmiwaC",
+ "tiw/PQOG0vuW4OgDKrAt2soAe6axoxdneLRiPRsZm2nTjrmoIFK5xKnr1dY2Tjugr6/cvZfLducuTAqx",
+ "22kOo7lMsWH5m7czDORe+prKCt7XqYwZVtEU3mZlGgNt0Uz/PFuTdTI0ZsY3WbbjCV9W/WsYXdstjK4j",
+ "zsYJNd62x7TSdK/h8l6/4vg3rMi9jcNoZ9gd+c8pLFvW2HdfugU/weAznGa+GIMvISF/qJZfmBkVadpI",
+ "y/3Gg9BsBPYR5NPIjSMqERWCgG9jlJA+ujACRGpbYkzFDN4HITgM5QlDqr6GYwDFl5aNzoY32s4YERc6",
+ "D3NoaWbmNAurP4vZZhVx5x/Myjdkj3PmMmhYKhTMxm1EKAy3X3yzZc2NkZJRZoy2ZmmiaM94GJ79i8i9",
+ "veU93nJ1ylMWf7enU7PjRq7Mc5HTdFQNejgviBKU3DlL0ZkwLUQVGi2MFw3hOBZEyqrQ+Y2oon22efY2",
+ "ydPq+Rq+P9H5D0PnvxHjhMmorYnC9T4FnDH6tCT1acLq4eoJH7N4wP2jtLZ+eaqsewVeA+jToEc/pcOI",
+ "niMjwxOYoIbiA2RpAFoDVXY7/7enR+wZCHp2ffVJS8P46DQq6vIkBmtn71rPkqG5eZ4fhsVCgoYTc+TO",
+ "fDWEAiqMR2pN8ud7ZFjjm3B+SOI4od6BWjRDQ0DlTQbAXSAG9KxzqIlV3QEoLGI1mn9yhTR1cO/QRdkV",
+ "xLOSIEAzzPCEyKr7xSfdPjrxdR+KMNPbPSIogoL4MTiMrII0+WvdKrQhbcP+ze3PM+uFPXSgIrsC6zvK",
+ "wQb4MsjB/WarqJS8s+CCm1HlghUXPEUxZz8pdI+ZeVQOYGZ+QsJiCCTN/Ty2AfztuyjfAHqMk/Hi5Pz1",
+ "4dEJulQCKzJZPLMQm4Abk9TF878h2/4SXHDZQyvTYkvjzzy1uyPgirzi+m87xEGWBPbw9WsUpULkjsnz",
+ "1+8vweunT3T6J9tbkBm/I5UBYJ2Z1xFw47DygZB5qbNzFh4YJGFmHYJmRX/eOFRnifKsaxEu0xKC70Kk",
+ "lDJLJhvw/a7Ri+tOFH1/TXLK76VNc8NwYsN+TDBEfmDyl1vwMRaHbPQx5j5FjaQTE0SEeugQqEJjBRB0",
+ "5BPDAfrzuuM5ta87N9fMUtK1e1dz3bG7WdOl6ze9uWYXcKm9vF0RzAsyT3BE0GGSmC2sgTYbQQ8XAN1y",
+ "ouLoL00C1x3Pqx9YSeFrEfjiJ7RlZwYq8Ca2tA+ZpYp2jhFMj+Z0tVWlmpyunnxX3ErDb+NnXXb4cJL6",
+ "X9DT+Hc4iRharBAddry3shdxYEM0zKPir7CCglfQx2bwzPlYtA+6aC7InbZBtI4uRa4ZCxYrnC8OdMRc",
+ "ENAPsX8HHXA1mpkf9+R/WM7RX+MAsFExT0z4g7q9zP49zG0/MNFRm+G3C9Izw2uO08xFeSqTRU5xAS7s",
+ "awNLcaH7FIMoIjzHI5pQtWjmtRP23bKaDUV74rQfk9MMZbVgtCgrldcmFspWL3WJCryqLrWuNeh/5N4v",
+ "bcLeLBR4rjE2Gy/3Ix+6JrNzZ80w21LKAZjN97/N9fZ36eryL5IzEnCsYuvpBnhl8JnGX2pjzvMbGexS",
+ "PkaObUYLmwoteNfi8ciGdIBXcrqW4n7gO5bv9QIEe/sPb9PraGxVd2qe9HG5N9VPjeRLcU38tcLbdH4E",
+ "4d3sKXDCe2UPQZDC/y4H+e/4lI1XlqeDyC+ZvYJkzcppW+FqabtOxrrmvy6O8iyaGxO4eR3wkMDNQH+S",
+ "uZuQuT5lZFThU2ReS3uTAviSKDd9DtJy0XtZT6wbEMJ+WfHlQthGlrk3hJFfX3yZXN5rKomvR30Sy+vn",
+ "iMsiRzQzQVkyZ5mOlgtmSD5l3z/lSWBGCy+HRVAyF3LKPI5oLqaxabCHs2XohT0R5BpfZ6lCqqBSAmpH",
+ "oNkerUtKB70c7001WuzSWUIqE5vM0Fyqlt51eSXwyyazHuk4zw5WL68heHOOhRqMuZj1YDioDRjxmLKJ",
+ "qwbt9bqyaY88qjcJVcjHeQI5A+ABSbcj1QKeW+uBzbZkTFEtNx1OwOBQYAr1+ukXRpRhiBcuJKZ6sfti",
+ "b+flcK8xn0tDjscolaqYvSVPO+GopJo+xpQJzh/w9n9ZmgQLllzNbFWVALZEMSBipLfvQaeOLC+wGY7E",
+ "TzJkXWcNy7OwoY0So06lDUbOZlt64nBqDXrYvZSZCZQX98gEWL2G+1UPQZZmDHhj3st72ZpN+UlId6xS",
+ "URe+nwCNBF9RD9f9Lv9tFTb5gc5rIOPjsSQ1oG2vJb/LJl8hwLYZolv29rtAJE/svj6Tocp9395k+MfZ",
+ "OcIimtK7QvZoo90Tgs6PT8tmDryLNGvJM+AvMSWA/L7WnrBwNpsUn+j8a0wKb456q2JEMn2IsCxisb2t",
+ "sUzVuwHbaPuabSzsXgsLYLgJedN0W/urxxKQ+nquSOwyeOWk9SSC1mZx1HA2liUifrBBMvgM/7m190x1",
+ "bxWPMItIopnHF4tITbGCNAsmUjFZNMkYM0ZVxizz2RSoLoJBytECP2KwwN/h6t9saYkqlqnHJfavcf7p",
+ "DYanhpWSi55HvjRrxQW/hNY2Yq81xHYWCNn6OJ8o99s57guiLEudtRHTrru0LRAHWIFlgW1KFqwei9bC",
+ "sDxjVFG4szV1+/VEToRXXadXgk4mRJwYeDZzPWAGtzPV3Q9YQK19ZJIdPWpITV6mJgChWQJSZg1Pt78b",
+ "4V9LISbWwNKj41yzAfUWkFdisE4XQQYenOR1Akck4WwCD3d4dhDsm2RDLgVRVoSQ5JUJf5LXLFSPMC8/",
+ "2dXz8Ps8WEK2qS8YrmYICYmgpCE2SYm84ofX7I9ysVjk14rt+mCbkdwzqGrptGu2VSoFaB+cjUiCRoLg",
+ "DzG/Z89skgocxxLxOyI0NSPJkZzyNLHPyEbkmqWyVMsWosgZITGJTenEqovLiLdTu5dLHFywdEgs0bAo",
+ "k+7RQ0Jlja7Q2rP+NTs27iTYLZNLCQ5CRIC5wiJNGFhyJm3tx4CTytURDnqp7Fn4UbNfeFU8G1xOfh3P",
+ "YsmFJ1PmW3mramqrOpFoNnYtTqqyNLWGU7sbW/BGWcO+VRTNJbR9pBAaEzdkZmwOlrVLeIqmWbtRbsnB",
+ "I5TNRDHmZNzCEMiuZLRJ4LS0TUho3un3G3SU3DzR6smaPHdOdDvYheXKJ+N0YxLZI5UlsYmZR65ltLfy",
+ "QqSgZlhmpliRSuOuV7PZxQ/orZ4iLK+ZURbd7AK8a6waaQxUF3hgn75DOhNFPhYKF5nyVyHTzHO7rG6U",
+ "yWUzZllAHa2HDSvd2RZb+25sK4eVEyZoNNXU+6W+sO+PnH7o7+IeikubkXPXer1Efsm5G18eyMHnvIDa",
+ "l4Hh0TYRBEliGdo46OFAOCKEhcsKdhEkQzd1wmZcKs1rcESlQqp+E4e/NiB9JdcU7/TyZbYq4OYXVqzU",
+ "zA9U4Fx6OedUpYHjiQO/ZWR1TsjFGnoeA8L+NzBfQ5lqCNEG9QHFrjPtUahaWJ+XcrUqjvVOWKjqCoE/",
+ "TfVazWlb4OiD1ume8wKSqlS5VI9qy51vxlNbqUhb46zN16IhelQ3bUE2BDi9UGr1h399/Hfg+kNbuThc",
+ "sLiG57XONE7gZgvapvcDl5VNrWMyEmkj2vmRAy/njTtZucdgG7IPzTQN14f2XmGKpVHmegzIzfp0bFtn",
+ "Qjp7uZSJ1bpbhVXNPNM/M/IgQ1YvP+C09aAVDTSSxP4hqaglw8/SDOVDBZ+TrGfnYXqzdOzamOLcnMO7",
+ "iIYmF4rLxlVG+pOu+JbewppN8V3fxS1etSRPhcfgcYb0yzybzHgCL8IFn+sSVVRZcBNmWoXCLemEjbXK",
+ "au3Dk8ez2VbgyNMysH+bCg1/i9gwP+fFQ/gzqCUHUyoVF4uWbhBb6RH8pGUYynz6AD9ICfbfLWx/Y2Va",
+ "9NU49Lb21pQQ5sryrsVxUxEHllSeJMA3VNAVhWJJxm3OwwXBXV7SuX0OBzd7QBy0MZ5LIP66cAT8HXN8",
+ "NwSMw4N98GXeeQG6amC7yxbaAjDK1PM9/1HVq+Fwd/fFcHv3+cv9vRcvni+vG/udGP2nISIpvsLzqoY/",
+ "SZnvRcqMFiin2HYCJg8KbHUsMBFimhy4mGBGP1E2yV9o1Nr80G1Tpj4M/hXJ6cY+dI9k7pspQ6xnMPyU",
+ "lu6xTHS3+XUxYx6XDD6bf5wZb+tS3ZcFu4YvXNxg675uaU5T5oXVanCgxHeVby8gQuIR+NZMVMe3nsyx",
+ "uHu8vOVLeVQA6E/n7u+CqQ0d+dJ8NYbOww9anrHzZ4n3VE0p8+1tF2mURyO5mKM8Xlz2r9m7POaodShQ",
+ "Fq2dmezXzI/Ym0wVEb17ov+BjPRAaooZ+u3kChWDrtAW+WhntcUNaGxKHfDZPFWuaK58VhPxZDB77L3d",
+ "XXvgU2W131noU9ExUKChVp6BLAqGOXFT9gl8VTRHTqSUWap8klDfQXAV7EcLWdX9Po2MGhHqHnYsF6Be",
+ "IMdc8AlUiPKfowTx1K+XQW+yJyUbNgfcRPVWgV3GE5t9y4OxNRkrb51+TFabcpPXqzFaX2vS36/evEYz",
+ "wlI0xxPzMsuvMG9qLITrkP7OZ+QcT1rEemhVO5iqWVLknSJUAIntlHmP9DoMZBpGPwkJDFdKQbL3cv9F",
+ "IfPYf/d//rdA7rEKI/6eTaOheDo0r42tgnuYVT5I1bRoZJsafmrRinSrdPqTtCXSssJnUHQ3K+nmm9dQ",
+ "KUF/kObeayIwU+ZKC6xH3b+PDtkiMA+KMEORNunVlEpEWDzn1FS7i6mM+B0B6qUC8XtmIAly0Jlb7gaV",
+ "kJtjaXERW5SIxQW0PL1/+ZpS++uJd9ODNjyjPzVZQLyriSDxPzBgxNTYsUQULs1VYWNzWh18hv++xTPy",
+ "NWflUmS+wlBVEYJLvPMzTNUYgi9/9aJ8v4Pz26VF96aOb7YeDJU5bp5Ydn3h9iVEl0nxqwLvTVCvzbga",
+ "MBszvmq0Gwum0XbBLPrzsPcP3Pu03Xt1e/NLyEIK3oVmD5DhJT9NFBGFNGwhL0tUTJIehrRNUR5n2SZ8",
+ "Quvvsot5WqAtgocCowUSJKaCRMrmpnD1ic+Oz21DbaMEnqfChCWhsWvke1mCmQn80fORi/z3mtvHAFWv",
+ "+8Vrl+U4OEjoYCBo0RQebu+9LGz4VKm5PBgM+j8HU/E+SYNGaaDZITJ7YYWCphEGpzFXQX8uuEliXHrh",
+ "Cltn99NTLM36U3fqaQNzhKMPtcT+O2ZxYkn9nR5liFwfFKfCWbmWemyyInRBIkJdkLRDsS2Rx2NirAaf",
+ "ObTOJx9NxVyJqD2c8Q+EyRpuOXKQL5F1h82z14kUHrcWfBVGyCXf9XWvrfCDJw1etXGNgKPLi1ONVEVc",
+ "nF4IVqlClXFqpfRwb2Vgb1aVTB4RVitMJ1bYrSKpnFytH+3hIut/1smsfyXpUxAoZUZ3h89lAoWn9cUi",
+ "XvOJEQdg7PPUvI83iMNZEXZJ4ExozgBhXWrmyWAKigcNyFdoUz0+OBO4KJBy5mX4OupNHIBPNLuhlA9m",
+ "C0HOt9aJMLq4C2uRc8Hj1FwBmkadbicVCfgIjdUT8+jToh/x2eBuB5L82WmqdQ8srUokSALuHsW93BRW",
+ "qBdTU1TVRc0wrqqLP1K50kvbwXyz245VeWC/6lhZaiTIbWFitdy+2DmKHvCVJ7BPlPX42Qte91I3n6Rw",
+ "Xlp9EeXgdQ/6cFhb2ynMezdvvOJDt7bDFMsS56P5JN92LJDXM8zwBJI7mgc28YwyKpUoj+/Xmf1y8+X/",
+ "BQAA//+IlZPWJEMBAA==",
}
// GetSwagger returns the content of the embedded swagger specification file
diff --git a/api/queryAPI/collector.go b/api/queryAPI/collector.go
index 8e350e46..e4f30816 100644
--- a/api/queryAPI/collector.go
+++ b/api/queryAPI/collector.go
@@ -6,59 +6,41 @@ import (
collectorset "queryorchestration/internal/collector/set"
- "github.com/google/uuid"
"github.com/labstack/echo/v4"
)
+// GetCollectorByClientId retrieves a client's collector configuration.
+// Query fields have been removed - only document processing settings remain.
func (s *Controllers) GetCollectorByClientId(ctx echo.Context, clientId ClientID) error {
coll, err := s.svc.Collector.GetByClientID(ctx.Request().Context(), clientId)
if err != nil {
return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to get collector: %s", err))
}
- fields := make([]CollectorField, len(coll.Fields))
- index := 0
- for name, id := range coll.Fields {
- fields[index] = CollectorField{
- Name: name,
- QueryId: id,
- }
- index++
- }
-
return ctx.JSON(http.StatusOK, Collector{
ClientId: clientId,
MinimumCleanerVersion: coll.MinCleanVersion,
MinimumTextVersion: coll.MinTextVersion,
ActiveVersion: coll.ActiveVersion,
LatestVersion: coll.LatestVersion,
- Fields: fields,
})
}
+// SetCollectorByClientId updates a client's collector configuration.
+// Query fields have been removed - only document processing settings remain.
func (s *Controllers) SetCollectorByClientId(ctx echo.Context, clientId ClientID) error {
req := CollectorSet{}
if err := ctx.Bind(&req); err != nil {
return echo.NewHTTPError(http.StatusBadRequest, err)
}
- var fields *map[string]uuid.UUID
- if req.Fields != nil {
- fs := map[string]uuid.UUID{}
- for _, field := range *req.Fields {
- fs[field.Name] = field.QueryId
- }
- fields = &fs
- }
-
err := s.svc.CollectorSet.SetByClientId(ctx.Request().Context(), clientId, &collectorset.SetParams{
ActiveVersion: req.ActiveVersion,
MinCleanVersion: req.MinimumCleanerVersion,
MinTextVersion: req.MinimumTextVersion,
- Fields: fields,
})
if err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to update query: %s", err))
+ return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to update collector: %s", err))
}
return ctx.NoContent(http.StatusOK)
diff --git a/api/queryAPI/collector_test.go b/api/queryAPI/collector_test.go
index 6608dc78..446fec7d 100644
--- a/api/queryAPI/collector_test.go
+++ b/api/queryAPI/collector_test.go
@@ -33,20 +33,12 @@ func TestSetCollector(t *testing.T) {
Name: "client_name",
})
require.NoError(t, err)
- queryId, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeContextFull)
- require.NoError(t, err)
av := int32(1)
cv := int64(1)
body := queryapi.CollectorSet{
ActiveVersion: &av,
MinimumCleanerVersion: &cv,
- Fields: &[]queryapi.CollectorField{
- {
- Name: "a",
- QueryId: queryId,
- },
- },
}
ctx, rec := createContextWithJSONBody(t, body)
@@ -85,6 +77,5 @@ func TestGetCollectorByClientId(t *testing.T) {
MinimumTextVersion: 0,
ActiveVersion: 0,
LatestVersion: 0,
- Fields: []queryapi.CollectorField{},
})
}
diff --git a/api/queryAPI/controllers.go b/api/queryAPI/controllers.go
index d3a15797..1af9fada 100644
--- a/api/queryAPI/controllers.go
+++ b/api/queryAPI/controllers.go
@@ -13,22 +13,18 @@ import (
"queryorchestration/internal/fieldextraction"
"queryorchestration/internal/folder"
"queryorchestration/internal/label"
- "queryorchestration/internal/query"
- querytest "queryorchestration/internal/query/test"
- queryupdate "queryorchestration/internal/query/update"
"queryorchestration/internal/serviceconfig/auth"
"queryorchestration/internal/serviceconfig/objectstore"
)
const Name = "queryAPI"
+// Services contains all service dependencies for the API controllers.
+// Query-related services have been removed - see remove_query_plan.md.
type Services struct {
Export *export.Service
Collector *collector.Service
CollectorSet *collectorset.Service
- Query *query.Service
- QueryUpdate *queryupdate.Service
- QueryTest *querytest.Service
Client *client.Service
ClientUpdate *clientupdate.Service
Document *document.Service
diff --git a/api/queryAPI/documents.go b/api/queryAPI/documents.go
index 14d7e66f..d11d55b4 100644
--- a/api/queryAPI/documents.go
+++ b/api/queryAPI/documents.go
@@ -93,7 +93,6 @@ func (s *Controllers) GetDocument(ctx echo.Context, id DocumentID, params GetDoc
Id: DocumentID(document.ID),
ClientId: ClientID(document.ClientID),
Hash: Hash(document.Hash),
- Fields: document.Fields,
HasTextRecord: document.HasTextRecord,
Labels: labels,
}
diff --git a/api/queryAPI/documents_test.go b/api/queryAPI/documents_test.go
index a06822ae..12051b30 100644
--- a/api/queryAPI/documents_test.go
+++ b/api/queryAPI/documents_test.go
@@ -117,7 +117,6 @@ func TestGetDocument(t *testing.T) {
Id: docId,
ClientId: "client_id",
Hash: "hash",
- Fields: map[string]interface{}{},
HasTextRecord: false,
Labels: []queryapi.LabelRecord{},
})
diff --git a/api/queryAPI/parse.go b/api/queryAPI/parse.go
index 7a9c5b79..45d9399c 100644
--- a/api/queryAPI/parse.go
+++ b/api/queryAPI/parse.go
@@ -4,69 +4,12 @@ import (
"errors"
"queryorchestration/internal/client"
- "queryorchestration/internal/query"
- resultprocessor "queryorchestration/internal/query/result/processor"
"github.com/google/uuid"
)
-func parseQueries(queries []*query.Query) ([]Query, error) {
- outQueries := make([]Query, len(queries))
- for index, query := range queries {
- qt, err := parseQuery(query)
- if err != nil {
- return nil, err
- }
-
- outQueries[index] = *qt
- }
-
- return outQueries, nil
-}
-
-func parseQuery(query *query.Query) (*Query, error) {
- if query.RequiredQueryIDs != nil && len(*query.RequiredQueryIDs) == 0 {
- query.RequiredQueryIDs = nil
- }
-
- qt, err := parseQueryType(query.Type)
- if err != nil {
- return nil, err
- }
-
- q := &Query{
- Id: query.ID,
- Type: qt,
- ActiveVersion: query.ActiveVersion,
- LatestVersion: query.LatestVersion,
- RequiredQueries: query.RequiredQueryIDs,
- Config: query.Config,
- }
-
- return q, nil
-}
-
-func parseQueryType(qType resultprocessor.Type) (QueryType, error) {
- switch qType {
- case resultprocessor.TypeJsonExtractor:
- return JSONEXTRACTOR, nil
- case resultprocessor.TypeContextFull:
- return CONTEXTFULL, nil
- default:
- return "", errors.New("invalid query type")
- }
-}
-
-func parseSpecQueryType(qType QueryType) (resultprocessor.Type, error) {
- switch qType {
- case JSONEXTRACTOR:
- return resultprocessor.TypeJsonExtractor, nil
- case CONTEXTFULL:
- return resultprocessor.TypeContextFull, nil
- default:
- return resultprocessor.Type(-1), errors.New("invalid query type")
- }
-}
+// Query parsing functions have been removed.
+// See remove_query_plan.md for details.
func parseStringToUUIDArray(sids *[]string) (*[]uuid.UUID, error) {
var ids []uuid.UUID
diff --git a/api/queryAPI/parse_test.go b/api/queryAPI/parse_test.go
index 20149c2e..57e0f1f6 100644
--- a/api/queryAPI/parse_test.go
+++ b/api/queryAPI/parse_test.go
@@ -4,116 +4,14 @@ import (
"testing"
"queryorchestration/internal/client"
- "queryorchestration/internal/query"
- resultprocessor "queryorchestration/internal/query/result/processor"
"github.com/google/uuid"
- "github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
-func TestParseQueries(t *testing.T) {
- cfg := "hey"
- in := []*query.Query{
- {
- ID: uuid.New(),
- Type: resultprocessor.TypeContextFull,
- ActiveVersion: 1,
- LatestVersion: 2,
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- },
- Config: &cfg,
- },
- }
- out, err := parseQueries(in)
- require.NoError(t, err)
- assert.Len(t, out, len(in))
- assert.ElementsMatch(t, []Query{
- {
- Id: in[0].ID,
- Type: CONTEXTFULL,
- ActiveVersion: 1,
- LatestVersion: 2,
- RequiredQueries: &[]types.UUID{
- (*in[0].RequiredQueryIDs)[0],
- },
- Config: in[0].Config,
- },
- }, out)
-}
-
-func TestParseQuery(t *testing.T) {
- cfg := "hey"
- in := &query.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeContextFull,
- ActiveVersion: 1,
- LatestVersion: 2,
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- },
- Config: &cfg,
- }
- out, err := parseQuery(in)
- require.NoError(t, err)
- assert.EqualExportedValues(t, Query{
- Id: in.ID,
- Type: CONTEXTFULL,
- ActiveVersion: 1,
- LatestVersion: 2,
- RequiredQueries: &[]types.UUID{
- (*in.RequiredQueryIDs)[0],
- },
- Config: in.Config,
- },
- *out)
-}
-
-func TestParseQueryMinimal(t *testing.T) {
- in := &query.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeContextFull,
- ActiveVersion: 1,
- LatestVersion: 2,
- }
- out, err := parseQuery(in)
- require.NoError(t, err)
- assert.EqualExportedValues(t, Query{
- Id: in.ID,
- Type: CONTEXTFULL,
- ActiveVersion: 1,
- LatestVersion: 2,
- },
- *out)
-}
-
-func TestParseQueryType(t *testing.T) {
- qt, err := parseQueryType(resultprocessor.TypeContextFull)
- require.NoError(t, err)
- assert.Equal(t, CONTEXTFULL, qt)
-
- qt, err = parseQueryType(resultprocessor.TypeJsonExtractor)
- require.NoError(t, err)
- assert.Equal(t, JSONEXTRACTOR, qt)
-
- _, err = parseQueryType(resultprocessor.Type(-1))
- assert.Error(t, err)
-}
-
-func TestParseSpecQueryType(t *testing.T) {
- qt, err := parseSpecQueryType(CONTEXTFULL)
- require.NoError(t, err)
- assert.Equal(t, resultprocessor.Type(resultprocessor.TypeContextFull), qt)
-
- qt, err = parseSpecQueryType(JSONEXTRACTOR)
- require.NoError(t, err)
- assert.Equal(t, resultprocessor.Type(resultprocessor.TypeJsonExtractor), qt)
-
- _, err = parseSpecQueryType("invalid")
- assert.Error(t, err)
-}
+// Query parsing tests have been removed.
+// See remove_query_plan.md for details.
func TestParseStringToUUIDArray(t *testing.T) {
ids := []uuid.UUID{uuid.New()}
diff --git a/api/queryAPI/query.go b/api/queryAPI/query.go
deleted file mode 100644
index aca963bc..00000000
--- a/api/queryAPI/query.go
+++ /dev/null
@@ -1,105 +0,0 @@
-package queryapi
-
-import (
- "fmt"
- "net/http"
-
- "queryorchestration/internal/query/result"
- resultprocessor "queryorchestration/internal/query/result/processor"
-
- "github.com/labstack/echo/v4"
-)
-
-func (s *Controllers) ListQueries(ctx echo.Context) error {
- queries, err := s.svc.Query.List(ctx.Request().Context())
- if err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to list query: %s", err))
- }
-
- outQueries, err := parseQueries(queries)
- if err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to list query: %s", err))
- }
-
- return ctx.JSON(http.StatusOK, ListQueries{
- Queries: outQueries,
- })
-}
-
-func (s *Controllers) GetQuery(ctx echo.Context, id QueryID) error {
- query, err := s.svc.Query.Get(ctx.Request().Context(), id)
- if err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to get query: %s", err))
- }
-
- qt, err := parseQuery(query)
- if err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to get query: %s", err))
- }
-
- return ctx.JSON(http.StatusOK, qt)
-}
-
-func (s *Controllers) CreateQuery(ctx echo.Context) error {
- req := QueryCreate{}
- if err := ctx.Bind(&req); err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, err)
- }
-
- qt, err := parseSpecQueryType(req.Type)
- if err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, "Invalid Query Type")
- }
-
- id, err := s.svc.Query.Create(ctx.Request().Context(), &resultprocessor.Create{
- Type: qt,
- RequiredQueryIDs: req.RequiredQueries,
- Config: req.Config,
- })
- if err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to create query: %s", err))
- }
-
- return ctx.JSON(http.StatusCreated, IdMessage{
- Id: id,
- })
-}
-
-func (s *Controllers) UpdateQuery(ctx echo.Context, id QueryID) error {
- req := QueryUpdate{}
- if err := ctx.Bind(&req); err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, err)
- }
-
- err := s.svc.QueryUpdate.Update(ctx.Request().Context(), &resultprocessor.Update{
- ActiveVersion: req.ActiveVersion,
- Config: req.Config,
- RequiredQueryIDs: req.RequiredQueries,
- ID: id,
- })
- if err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to update query: %s", err))
- }
-
- return ctx.NoContent(http.StatusOK)
-}
-
-func (s *Controllers) TestQuery(ctx echo.Context, id QueryID) error {
- req := QueryTestRequest{}
- if err := ctx.Bind(&req); err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, err)
- }
-
- value, err := s.svc.QueryTest.Test(ctx.Request().Context(), result.Process{
- QueryID: id,
- QueryVersion: req.QueryVersion,
- DocumentID: req.DocumentId,
- })
- if err != nil {
- return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to test query: %s", err))
- }
-
- return ctx.JSON(http.StatusOK, QueryTestResponse{
- Value: value,
- })
-}
diff --git a/api/queryAPI/query_test.go b/api/queryAPI/query_test.go
deleted file mode 100644
index ef447d71..00000000
--- a/api/queryAPI/query_test.go
+++ /dev/null
@@ -1,306 +0,0 @@
-package queryapi_test
-
-import (
- "fmt"
- "net/http"
- "regexp"
- "testing"
- "time"
-
- "queryorchestration/internal/backgroundtask"
- "queryorchestration/internal/client"
- "queryorchestration/internal/collector"
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/document"
- "queryorchestration/internal/export"
- "queryorchestration/internal/query"
- "queryorchestration/internal/query/result"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/serviceconfig/queue"
- "queryorchestration/internal/serviceconfig/queue/clientsync"
- "queryorchestration/internal/serviceconfig/queue/queryversionsync"
- "queryorchestration/internal/test"
-
- queryapi "queryorchestration/api/queryAPI"
-
- clientupdate "queryorchestration/internal/client/update"
-
- collectorset "queryorchestration/internal/collector/set"
-
- documentbatch "queryorchestration/internal/document/batch"
- documentupload "queryorchestration/internal/document/upload"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
- querytest "queryorchestration/internal/query/test"
- queryupdate "queryorchestration/internal/query/update"
-
- "github.com/jackc/pgx/v5/pgtype"
- "github.com/stretchr/testify/require"
-
- "github.com/google/uuid"
- "github.com/stretchr/testify/assert"
-)
-
-func TestCreateQuery(t *testing.T) {
- cfg := &ControllerConfig{}
- test.CreateDB(t, cfg)
- initializeTestConfig(t, cfg)
-
- svc := createControllerServices(cfg)
- cons := queryapi.NewControllers(svc, cfg)
-
- body := queryapi.QueryCreate{
- Type: queryapi.CONTEXTFULL,
- }
- ctx, rec := createContextWithJSONBody(t, body)
-
- err := cons.CreateQuery(ctx)
- require.NoError(t, err)
- assert.Equal(t, http.StatusCreated, rec.Code)
- res := getBody[queryapi.IdMessage](t, rec)
- assert.NotEqual(t, uuid.Nil, res.Id)
-}
-
-func TestListQueries(t *testing.T) {
- cfg := &ControllerConfig{}
- test.CreateDB(t, cfg)
- initializeTestConfig(t, cfg)
-
- svc := createControllerServices(cfg)
- cons := queryapi.NewControllers(svc, cfg)
-
- ctx, rec := createContext(t)
-
- id, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeContextFull)
- require.NoError(t, err)
-
- err = cons.ListQueries(ctx)
- require.NoError(t, err)
- assert.Equal(t, http.StatusOK, rec.Code)
- assertBody(t, rec, queryapi.ListQueries{
- Queries: []queryapi.Query{
- {
- Id: id,
- Type: queryapi.CONTEXTFULL,
- ActiveVersion: 0,
- LatestVersion: 0,
- },
- },
- })
-}
-
-func TestGetQuery(t *testing.T) {
- cfg := &ControllerConfig{}
- test.CreateDB(t, cfg)
- initializeTestConfig(t, cfg)
-
- svc := createControllerServices(cfg)
- cons := queryapi.NewControllers(svc, cfg)
-
- ctx, rec := createContext(t)
-
- id, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeContextFull)
- require.NoError(t, err)
-
- err = cons.GetQuery(ctx, id)
- require.NoError(t, err)
- assert.Equal(t, http.StatusOK, rec.Code)
- assertBody(t, rec, queryapi.Query{
- Id: id,
- Type: queryapi.CONTEXTFULL,
- ActiveVersion: 0,
- LatestVersion: 0,
- })
-}
-
-func TestUpdateQuery(t *testing.T) {
- cfg := &ControllerConfig{}
- test.CreateDB(t, cfg)
- initializeTestConfig(t, cfg)
- acfg := test.CreateAWSContainer(t, cfg)
-
- test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
- cfg.QueryVersionSyncURL = test.CreateQueue(t, cfg, test.QueryVersionSyncRunnerName)
-
- svc := createControllerServices(cfg)
- cons := queryapi.NewControllers(svc, cfg)
-
- id, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeJsonExtractor)
- require.NoError(t, err)
-
- av := int32(1)
- c := `{"path": "val"}`
- body := queryapi.QueryUpdate{
- ActiveVersion: &av,
- Config: &c,
- }
- ctx, rec := createContextWithJSONBody(t, body)
-
- err = cons.UpdateQuery(ctx, id)
- require.NoError(t, err)
- assert.Equal(t, http.StatusOK, rec.Code)
- assert.Empty(t, rec.Body.String())
-
- test.AssertMessageBody(t, cfg, cfg.GetQueryVersionSyncURL(), regexp.MustCompile(fmt.Sprintf(`{"id":"%s"}`, id)))
-}
-
-type ControllerConfig struct {
- serviceconfig.BaseConfig
- queryversionsync.QueryVersionSyncConfig
- clientsync.ClientSyncConfig
- queue.QueueConfig
- objectstore.ObjectStoreConfig
-}
-
-func (c *ControllerConfig) GetBackgroundRunner() *backgroundtask.Runner {
- return nil // Test config returns nil
-}
-
-func createControllerServices(cfg *ControllerConfig) *queryapi.Services {
- docsvc := document.New(cfg)
- col := collector.New(cfg)
- que := query.New(cfg)
- cli := client.New(cfg)
- return &queryapi.Services{
- Export: export.New(),
- Collector: col,
- CollectorSet: collectorset.New(cfg, &collectorset.Services{
- Collector: col,
- }),
- Query: que,
- QueryUpdate: queryupdate.New(cfg, &queryupdate.Services{
- Query: que,
- }),
- QueryTest: querytest.New(cfg, &querytest.Services{
- Result: result.New(cfg, &result.Services{
- Query: que,
- }),
- Collector: col,
- Document: docsvc,
- }),
- Client: cli,
- ClientUpdate: clientupdate.New(cfg, &clientupdate.Services{
- Client: cli,
- }),
- Document: docsvc,
- DocumentUpload: documentupload.New(cfg),
- DocumentBatch: documentbatch.New(cfg),
- }
-}
-
-func createDocumentWithText(t testing.TB, cfg *ControllerConfig, clientId string) (uuid.UUID, uuid.UUID, uuid.UUID) {
- docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{
- Clientid: clientId,
- Hash: "hash",
- })
- require.NoError(t, err)
- fill := "fill"
- cleanId, err := cfg.GetDBQueries().AddDocumentClean(t.Context(), &repository.AddDocumentCleanParams{
- Documentid: docId,
- Bucket: &fill,
- Key: &fill,
- Hash: &fill,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentCleanEntry(t.Context(), &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanId,
- Version: 1,
- })
- require.NoError(t, err)
- textId, err := cfg.GetDBQueries().AddDocumentText(t.Context(), &repository.AddDocumentTextParams{
- Cleanid: cleanId,
- Bucket: fill,
- Key: fill,
- Hash: fill,
- Createdat: pgtype.Timestamp{
- Time: time.Now(),
- Valid: true,
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentTextEntry(t.Context(), &repository.AddDocumentTextEntryParams{
- Textid: textId,
- Version: 1,
- })
- require.NoError(t, err)
-
- return docId, cleanId, textId
-}
-
-func TestTestQuery(t *testing.T) {
- cfg := &ControllerConfig{}
- test.CreateDB(t, cfg)
- initializeTestConfig(t, cfg)
-
- svc := createControllerServices(cfg)
- cons := queryapi.NewControllers(svc, cfg)
-
- contextQueryId, err := svc.Query.Create(t.Context(), &resultprocessor.Create{
- Type: resultprocessor.TypeContextFull,
- })
- require.NoError(t, err)
-
- _, err = svc.Client.Create(t.Context(), client.CreateParams{
- ID: "client_id",
- Name: "client_name",
- })
- require.NoError(t, err)
-
- docId, _, textId := createDocumentWithText(t, cfg, "client_id")
-
- strVal := `{"mykey": "example_value", "oldkey": "old_value"}`
- _, err = cfg.GetDBQueries().AddResult(t.Context(), &repository.AddResultParams{
- Queryid: contextQueryId,
- Value: strVal,
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- c := `{"path": "oldkey"}`
- queryId, err := svc.Query.Create(t.Context(), &resultprocessor.Create{
- Type: resultprocessor.TypeJsonExtractor,
- Config: &c,
- RequiredQueryIDs: &[]uuid.UUID{
- contextQueryId,
- },
- })
- require.NoError(t, err)
- c = `{"path": "mykey"}`
- err = svc.QueryUpdate.Update(t.Context(), &resultprocessor.Update{
- ID: queryId,
- Config: &c,
- })
- require.NoError(t, err)
-
- body := queryapi.QueryTestRequest{
- DocumentId: docId,
- QueryVersion: 1,
- }
- ctx, rec := createContextWithJSONBody(t, body)
-
- err = cons.TestQuery(ctx, queryId)
- require.NoError(t, err)
- assert.Equal(t, http.StatusOK, rec.Code)
- assertBody(t, rec, queryapi.QueryTestResponse{
- Value: "old_value",
- })
-
- body = queryapi.QueryTestRequest{
- DocumentId: docId,
- QueryVersion: 2,
- }
- ctx, rec = createContextWithJSONBody(t, body)
-
- err = cons.TestQuery(ctx, queryId)
- require.NoError(t, err)
- assert.Equal(t, http.StatusOK, rec.Code)
- assertBody(t, rec, queryapi.QueryTestResponse{
- Value: "example_value",
- })
-}
diff --git a/api/queryAPI/testutils_test.go b/api/queryAPI/testutils_test.go
new file mode 100644
index 00000000..153bc37b
--- /dev/null
+++ b/api/queryAPI/testutils_test.go
@@ -0,0 +1,66 @@
+package queryapi_test
+
+import (
+ queryapi "queryorchestration/api/queryAPI"
+ "queryorchestration/internal/backgroundtask"
+ "queryorchestration/internal/client"
+ clientupdate "queryorchestration/internal/client/update"
+ "queryorchestration/internal/collector"
+ collectorset "queryorchestration/internal/collector/set"
+ "queryorchestration/internal/document"
+ documentbatch "queryorchestration/internal/document/batch"
+ documentupload "queryorchestration/internal/document/upload"
+ "queryorchestration/internal/export"
+ "queryorchestration/internal/fieldextraction"
+ "queryorchestration/internal/folder"
+ "queryorchestration/internal/label"
+ "queryorchestration/internal/server/api"
+ "queryorchestration/internal/serviceconfig/queue/clientsync"
+)
+
+// ControllerConfig provides test configuration for API controllers.
+// Note: Query-related configurations have been removed. See remove_query_plan.md.
+type ControllerConfig struct {
+ api.BaseConfig
+ clientsync.ClientSyncConfig
+ BackgroundRunner *backgroundtask.Runner
+}
+
+// GetBackgroundRunner returns the background runner for batch processing.
+func (c *ControllerConfig) GetBackgroundRunner() *backgroundtask.Runner {
+ return c.BackgroundRunner
+}
+
+// createControllerServices creates a Services struct for testing.
+// Note: Query-related services have been removed. See remove_query_plan.md.
+func createControllerServices(cfg *ControllerConfig) *queryapi.Services {
+ exp := export.New()
+ col := collector.New(cfg)
+ colupdate := collectorset.New(cfg, &collectorset.Services{
+ Collector: col,
+ })
+ cli := client.New(cfg)
+ cliUpdate := clientupdate.New(cfg, &clientupdate.Services{
+ Client: cli,
+ })
+ doc := document.New(cfg)
+ docup := documentupload.New(cfg)
+ docbatch := documentbatch.New(cfg)
+ fieldext := fieldextraction.New(cfg)
+ fld := folder.New(cfg)
+ lbl := label.New(cfg)
+
+ return &queryapi.Services{
+ Export: exp,
+ Collector: col,
+ CollectorSet: colupdate,
+ Client: cli,
+ ClientUpdate: cliUpdate,
+ Document: doc,
+ DocumentUpload: docup,
+ DocumentBatch: docbatch,
+ FieldExtraction: fieldext,
+ Folder: fld,
+ Label: lbl,
+ }
+}
diff --git a/api/queryRunner/runner.go b/api/queryRunner/runner.go
deleted file mode 100644
index a0f7c6c3..00000000
--- a/api/queryRunner/runner.go
+++ /dev/null
@@ -1,44 +0,0 @@
-package queryrunner
-
-import (
- "context"
- "log/slog"
-
- resultset "queryorchestration/internal/query/result/set"
-
- "github.com/google/uuid"
-)
-
-const Name = "queryRunner"
-
-type Services struct {
- ResultSet *resultset.Service
-}
-
-type Runner struct {
- svc *Services
-}
-
-func New(svc *Services) Runner {
- return Runner{
- svc: svc,
- }
-}
-
-type Body struct {
- DocumentID uuid.UUID `json:"document_id" validate:"required,uuid"`
- QueryID uuid.UUID `json:"query_id" validate:"required,uuid"`
-}
-
-func (s *Runner) Process(ctx context.Context, body Body) bool {
- err := s.svc.ResultSet.Set(ctx, &resultset.Set{
- DocumentID: body.DocumentID,
- QueryID: body.QueryID,
- })
- if err != nil {
- slog.Error("unable to process", "error", err)
- return false
- }
-
- return true
-}
diff --git a/api/queryRunner/runner_test.go b/api/queryRunner/runner_test.go
deleted file mode 100644
index a9a2b4b2..00000000
--- a/api/queryRunner/runner_test.go
+++ /dev/null
@@ -1,143 +0,0 @@
-package queryrunner_test
-
-import (
- "fmt"
- "regexp"
- "testing"
- "time"
-
- queryrunner "queryorchestration/api/queryRunner"
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/query"
- "queryorchestration/internal/query/result"
- resultset "queryorchestration/internal/query/result/set"
- resultsync "queryorchestration/internal/query/result/sync"
- "queryorchestration/internal/server/runner"
- "queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/serviceconfig/queue"
- queryc "queryorchestration/internal/serviceconfig/queue/query"
- "queryorchestration/internal/test"
-
- "github.com/jackc/pgx/v5/pgtype"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-type QueryConfig struct {
- runner.BaseConfig[queryrunner.Body]
- queryc.QueryConfig
- queue.QueueConfig
- objectstore.ObjectStoreConfig
-}
-
-func TestQueryRunner(t *testing.T) {
- cfg := &QueryConfig{}
- test.CreateDB(t, cfg)
- acfg := test.CreateAWSContainer(t, cfg)
-
- test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
- cfg.QueryURL = test.CreateQueue(t, cfg, test.QueryRunnerName)
-
- que := query.New(cfg)
- runner := queryrunner.New(&queryrunner.Services{
- ResultSet: resultset.New(cfg, &resultset.Services{
- Query: que,
- Result: result.New(cfg, &result.Services{
- Query: que,
- }),
- Sync: resultsync.New(cfg),
- }),
- })
-
- err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
- Clientid: "client_one",
- Name: "name_one",
- })
- require.NoError(t, err)
- docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{
- Clientid: "client_one",
- Hash: "hash",
- })
- require.NoError(t, err)
- fill := "fill"
- cleanId, err := cfg.GetDBQueries().AddDocumentClean(t.Context(), &repository.AddDocumentCleanParams{
- Documentid: docId,
- Bucket: &fill,
- Key: &fill,
- Hash: &fill,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentCleanEntry(t.Context(), &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanId,
- Version: 1,
- })
- require.NoError(t, err)
- textId, err := cfg.GetDBQueries().AddDocumentText(t.Context(), &repository.AddDocumentTextParams{
- Cleanid: cleanId,
- Bucket: fill,
- Key: fill,
- Hash: fill,
- Createdat: pgtype.Timestamp{
- Time: time.Now(),
- Valid: true,
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentTextEntry(t.Context(), &repository.AddDocumentTextEntryParams{
- Textid: textId,
- Version: 1,
- })
- require.NoError(t, err)
-
- contextQueryId, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeContextFull)
- require.NoError(t, err)
- version, err := cfg.GetDBQueries().AddLatestQueryVersion(t.Context(), contextQueryId)
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddActiveQueryVersion(t.Context(), &repository.AddActiveQueryVersionParams{
- Versionid: version,
- Queryid: contextQueryId,
- })
- require.NoError(t, err)
- queryId, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeJsonExtractor)
- require.NoError(t, err)
- version, err = cfg.GetDBQueries().AddLatestQueryVersion(t.Context(), queryId)
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddActiveQueryVersion(t.Context(), &repository.AddActiveQueryVersionParams{
- Versionid: version,
- Queryid: queryId,
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddRequiredQuery(t.Context(), &repository.AddRequiredQueryParams{
- Queryid: queryId,
- Requiredqueryid: contextQueryId,
- Addedversion: version,
- })
- require.NoError(t, err)
- collectorOneVersion, err := cfg.GetDBQueries().AddLatestCollectorVersion(t.Context(), "client_one")
- require.NoError(t, err)
- err = cfg.GetDBQueries().SetActiveCollectorVersion(t.Context(), &repository.SetActiveCollectorVersionParams{
- Versionid: version,
- Clientid: "client_one",
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddCollectorQuery(t.Context(), &repository.AddCollectorQueryParams{
- Queryid: queryId,
- Clientid: "client_one",
- Name: "EXAMPLE_ONE",
- Addedversion: collectorOneVersion,
- })
- require.NoError(t, err)
-
- doc := queryrunner.Body{
- DocumentID: docId,
- QueryID: contextQueryId,
- }
-
- assert.True(t, runner.Process(t.Context(), doc))
-
- test.AssertMessageBody(t, cfg, cfg.GetQueryURL(), regexp.MustCompile(fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", docId, queryId)))
-}
diff --git a/api/querySyncRunner/runner.go b/api/querySyncRunner/runner.go
deleted file mode 100644
index d253089f..00000000
--- a/api/querySyncRunner/runner.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package querysyncrunner
-
-import (
- "context"
- "log/slog"
-
- querysync "queryorchestration/internal/query/sync"
-
- "github.com/google/uuid"
-)
-
-const Name = "querySyncRunner"
-
-type Services struct {
- QuerySync *querysync.Service
-}
-
-type Runner struct {
- svc *Services
-}
-
-func New(svc *Services) Runner {
- return Runner{
- svc: svc,
- }
-}
-
-type Body struct {
- DocumentID uuid.UUID `json:"id" validate:"required,uuid"`
-}
-
-func (s *Runner) Process(ctx context.Context, body Body) bool {
- err := s.svc.QuerySync.Sync(ctx, body.DocumentID)
- if err != nil {
- slog.Error("unable to process", "error", err)
- return false
- }
-
- return true
-}
diff --git a/api/querySyncRunner/runner_test.go b/api/querySyncRunner/runner_test.go
deleted file mode 100644
index 4ca51edf..00000000
--- a/api/querySyncRunner/runner_test.go
+++ /dev/null
@@ -1,120 +0,0 @@
-package querysyncrunner_test
-
-import (
- "fmt"
- "regexp"
- "testing"
- "time"
-
- querysyncrunner "queryorchestration/api/querySyncRunner"
- "queryorchestration/internal/database/repository"
- resultsync "queryorchestration/internal/query/result/sync"
- querysync "queryorchestration/internal/query/sync"
- "queryorchestration/internal/server/runner"
- "queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/serviceconfig/queue"
- queryc "queryorchestration/internal/serviceconfig/queue/query"
- "queryorchestration/internal/test"
-
- "github.com/jackc/pgx/v5/pgtype"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-type QuerySyncConfig struct {
- runner.BaseConfig[querysyncrunner.Body]
- queryc.QueryConfig
- queue.QueueConfig
- objectstore.ObjectStoreConfig
-}
-
-func TestQuerySyncRunner(t *testing.T) {
- cfg := &QuerySyncConfig{}
- test.CreateDB(t, cfg)
- acfg := test.CreateAWSContainer(t, cfg)
-
- test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
- cfg.QueryURL = test.CreateQueue(t, cfg, test.QueryRunnerName)
-
- runner := querysyncrunner.New(&querysyncrunner.Services{
- QuerySync: querysync.New(cfg, &querysync.Services{
- ResultSync: resultsync.New(cfg),
- }),
- })
-
- err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
- Clientid: "client_one",
- Name: "name_one",
- })
- require.NoError(t, err)
- docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{
- Clientid: "client_one",
- Hash: "hash",
- })
- require.NoError(t, err)
- fill := "fill"
- cleanId, err := cfg.GetDBQueries().AddDocumentClean(t.Context(), &repository.AddDocumentCleanParams{
- Documentid: docId,
- Bucket: &fill,
- Key: &fill,
- Hash: &fill,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentCleanEntry(t.Context(), &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanId,
- Version: 1,
- })
- require.NoError(t, err)
- textId, err := cfg.GetDBQueries().AddDocumentText(t.Context(), &repository.AddDocumentTextParams{
- Cleanid: cleanId,
- Bucket: fill,
- Key: fill,
- Hash: fill,
- Createdat: pgtype.Timestamp{
- Time: time.Now(),
- Valid: true,
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentTextEntry(t.Context(), &repository.AddDocumentTextEntryParams{
- Textid: textId,
- Version: 1,
- })
- require.NoError(t, err)
-
- queryId, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeContextFull)
- require.NoError(t, err)
- version, err := cfg.GetDBQueries().AddLatestQueryVersion(t.Context(), queryId)
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddActiveQueryVersion(t.Context(), &repository.AddActiveQueryVersionParams{
- Versionid: version,
- Queryid: queryId,
- })
- require.NoError(t, err)
- collectorOneVersion, err := cfg.GetDBQueries().AddLatestCollectorVersion(t.Context(), "client_one")
- require.NoError(t, err)
- err = cfg.GetDBQueries().SetActiveCollectorVersion(t.Context(), &repository.SetActiveCollectorVersionParams{
- Versionid: version,
- Clientid: "client_one",
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddCollectorQuery(t.Context(), &repository.AddCollectorQueryParams{
- Queryid: queryId,
- Clientid: "client_one",
- Name: "EXAMPLE_ONE",
- Addedversion: collectorOneVersion,
- })
- require.NoError(t, err)
-
- doc := querysyncrunner.Body{
- DocumentID: docId,
- }
-
- assert.True(t, runner.Process(t.Context(), doc))
-
- test.AssertMessageBody(t, cfg, cfg.GetQueryURL(), regexp.MustCompile(fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", docId, queryId)))
-}
diff --git a/api/queryVersionSyncRunner/runner.go b/api/queryVersionSyncRunner/runner.go
deleted file mode 100644
index 3318b14c..00000000
--- a/api/queryVersionSyncRunner/runner.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package queryversionsyncrunner
-
-import (
- "context"
- "log/slog"
-
- queryversionsync "queryorchestration/internal/query/versionsync"
-
- "github.com/google/uuid"
-)
-
-const Name = "queryVersionSyncRunner"
-
-type Services struct {
- Sync *queryversionsync.Service
-}
-
-type Runner struct {
- svc *Services
-}
-
-func New(svc *Services) Runner {
- return Runner{
- svc: svc,
- }
-}
-
-type Body struct {
- QueryID uuid.UUID `json:"id" validate:"required,uuid"`
-}
-
-func (s *Runner) Process(ctx context.Context, body Body) bool {
- err := s.svc.Sync.Sync(ctx, body.QueryID)
- if err != nil {
- slog.Error("unable to process", "error", err)
- return false
- }
-
- return true
-}
diff --git a/api/queryVersionSyncRunner/runner_test.go b/api/queryVersionSyncRunner/runner_test.go
deleted file mode 100644
index 282ecb6b..00000000
--- a/api/queryVersionSyncRunner/runner_test.go
+++ /dev/null
@@ -1,99 +0,0 @@
-package queryversionsyncrunner_test
-
-import (
- "regexp"
- "testing"
-
- queryversionsyncrunner "queryorchestration/api/queryVersionSyncRunner"
- "queryorchestration/internal/database/repository"
- queryversionsync "queryorchestration/internal/query/versionsync"
- "queryorchestration/internal/server/runner"
- "queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/serviceconfig/queue"
- "queryorchestration/internal/serviceconfig/queue/clientsync"
- "queryorchestration/internal/test"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-type QuerySyncConfig struct {
- runner.BaseConfig[queryversionsyncrunner.Body]
- clientsync.ClientSyncConfig
- queue.QueueConfig
- objectstore.ObjectStoreConfig
-}
-
-func TestQueryVersionSyncRunner(t *testing.T) {
- cfg := &QuerySyncConfig{}
- test.CreateDB(t, cfg)
- acfg := test.CreateAWSContainer(t, cfg)
-
- test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
- cfg.ClientSyncURL = test.CreateQueue(t, cfg, test.ClientSyncRunnerName)
-
- svc := queryversionsync.New(cfg)
-
- runner := queryversionsyncrunner.New(&queryversionsyncrunner.Services{
- Sync: svc,
- })
-
- queryId, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeContextFull)
- require.NoError(t, err)
- version, err := cfg.GetDBQueries().AddLatestQueryVersion(t.Context(), queryId)
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddActiveQueryVersion(t.Context(), &repository.AddActiveQueryVersionParams{
- Versionid: version,
- Queryid: queryId,
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
- Clientid: "client_one",
- Name: "name_one",
- })
- require.NoError(t, err)
- collectorOneVersion, err := cfg.GetDBQueries().AddLatestCollectorVersion(t.Context(), "client_one")
- require.NoError(t, err)
- err = cfg.GetDBQueries().SetActiveCollectorVersion(t.Context(), &repository.SetActiveCollectorVersionParams{
- Versionid: version,
- Clientid: "client_one",
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddCollectorQuery(t.Context(), &repository.AddCollectorQueryParams{
- Queryid: queryId,
- Clientid: "client_one",
- Name: "EXAMPLE_ONE",
- Addedversion: collectorOneVersion,
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
- Clientid: "client_two",
- Name: "name_two",
- })
- require.NoError(t, err)
- collectorTwoVersion, err := cfg.GetDBQueries().AddLatestCollectorVersion(t.Context(), "client_two")
- require.NoError(t, err)
- err = cfg.GetDBQueries().SetActiveCollectorVersion(t.Context(), &repository.SetActiveCollectorVersionParams{
- Versionid: version,
- Clientid: "client_two",
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddCollectorQuery(t.Context(), &repository.AddCollectorQueryParams{
- Queryid: queryId,
- Clientid: "client_two",
- Name: "EXAMPLE_TWO",
- Addedversion: collectorTwoVersion,
- })
- require.NoError(t, err)
-
- doc := queryversionsyncrunner.Body{
- QueryID: queryId,
- }
-
- assert.True(t, runner.Process(t.Context(), doc))
-
- test.AssertMessageBodies(t, cfg, cfg.GetClientSyncURL(), []*regexp.Regexp{
- regexp.MustCompile(`^{"id":"client_one"}$`),
- regexp.MustCompile(`^{"id":"client_two"}$`),
- })
-}
diff --git a/cmd/docTextRunner/main.go b/cmd/docTextRunner/main.go
index 3891959d..14c57d92 100644
--- a/cmd/docTextRunner/main.go
+++ b/cmd/docTextRunner/main.go
@@ -4,7 +4,8 @@
//
// If the document text is not already extracted according to the collector standards, trigger the textract job for text detection.
//
-// Otherwise, add an event to the QUERYSYNC queue.
+// This is the terminal step in the document processing pipeline.
+// Query processing has been removed - pipeline ends after text extraction.
package main
import (
@@ -17,7 +18,6 @@ import (
"queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/build"
"queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/textract"
_ "github.com/lib/pq"
@@ -26,7 +26,6 @@ import (
type DocTextConfig struct {
runner.BaseConfig[doctextrunner.Body]
textract.TextractConfig
- querysync.QuerySyncConfig
objectstore.ObjectStoreConfig
}
diff --git a/cmd/queryAPI/main.go b/cmd/queryAPI/main.go
index 763116f0..b2437c63 100644
--- a/cmd/queryAPI/main.go
+++ b/cmd/queryAPI/main.go
@@ -4,13 +4,13 @@
//
// - Client - A client is a single client of the Doczy project, all configurations and clients for a client will be in reference to the respective client.
//
-// - Collector - A collector specifies the required queries for export for a given client.
+// - Collector - A collector specifies document processing settings for a given client.
//
// - Sample - A sample is a subset of documents to be used for testing. When it is active a client will only process documents in the sample.
//
-// - Query - A query is a block of processing of data which takes an input and produces an output.
-//
// - Export - An export is a manually started process which outputs the results for a client to a given location.
+//
+// Note: Query functionality has been removed from this API. See remove_query_plan.md for details.
package main
import (
@@ -28,14 +28,11 @@ import (
"queryorchestration/internal/fieldextraction"
"queryorchestration/internal/folder"
"queryorchestration/internal/label"
- "queryorchestration/internal/query"
- "queryorchestration/internal/query/result"
"queryorchestration/internal/server/api"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/build"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/clientsync"
- "queryorchestration/internal/serviceconfig/queue/queryversionsync"
queryapi "queryorchestration/api/queryAPI"
documentbatch "queryorchestration/internal/document/batch"
@@ -45,9 +42,6 @@ import (
collectorset "queryorchestration/internal/collector/set"
- querytest "queryorchestration/internal/query/test"
- queryupdate "queryorchestration/internal/query/update"
-
"github.com/getkin/kin-openapi/openapi3"
_ "github.com/lib/pq"
)
@@ -55,7 +49,6 @@ import (
type QueryAPIConfig struct {
api.BaseConfig
clientsync.ClientSyncConfig
- queryversionsync.QueryVersionSyncConfig
objectstore.ObjectStoreConfig
BackgroundRunner *backgroundtask.Runner
}
@@ -70,10 +63,6 @@ func main() {
cfg.RegisterHandlersFunc = func() (*openapi3.T, error) {
exp := export.New()
- que := query.New(cfg)
- res := result.New(cfg, &result.Services{
- Query: que,
- })
col := collector.New(cfg)
colupdate := collectorset.New(cfg, &collectorset.Services{
Collector: col,
@@ -85,15 +74,6 @@ func main() {
doc := document.New(cfg)
docup := documentupload.New(cfg)
docbatch := documentbatch.New(cfg)
- quetest := querytest.New(cfg, &querytest.Services{
- Collector: col,
- Result: res,
- Document: doc,
- })
- qupdate := queryupdate.New(cfg, &queryupdate.Services{
- Query: que,
- })
-
fieldext := fieldextraction.New(cfg)
fld := folder.New(cfg)
lbl := label.New(cfg)
@@ -102,9 +82,6 @@ func main() {
Export: exp,
Collector: col,
CollectorSet: colupdate,
- Query: que,
- QueryUpdate: qupdate,
- QueryTest: quetest,
Client: cli,
ClientUpdate: cliUpdate,
Document: doc,
diff --git a/cmd/queryRunner/main.go b/cmd/queryRunner/main.go
index d48c166d..49140f0b 100644
--- a/cmd/queryRunner/main.go
+++ b/cmd/queryRunner/main.go
@@ -1,66 +1,59 @@
-// **Listens For**: Body Containing Document ID and Query ID
-//
-// **Action**.
-//
-// Run the query for the given document and store the result.
-//
-// Events with the document id and downstream query ids are pushed to the QUERY queue.
+// DEPRECATED: This runner is deprecated and no longer has any functionality.
+// It is left in place until deployment code is updated accordingly.
+// See remove_query_plan.md for details on the query removal.
package main
import (
"context"
"log/slog"
+ "net/http"
"os"
+ "os/signal"
+ "syscall"
+ "time"
- queryrunner "queryorchestration/api/queryRunner"
- "queryorchestration/internal/query"
- "queryorchestration/internal/query/result"
- resultset "queryorchestration/internal/query/result/set"
- resultsync "queryorchestration/internal/query/result/sync"
- "queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/build"
- queryc "queryorchestration/internal/serviceconfig/queue/query"
-
- _ "github.com/lib/pq"
)
-type QueryConfig struct {
- runner.BaseConfig[queryrunner.Body]
- queryc.QueryConfig
-}
+const serviceName = "queryRunner"
func main() {
- // Print version information before any environment checks
- build.PrintVersionInfo("queryRunner")
+ // Print version information
+ build.PrintVersionInfo(serviceName)
- ctx := context.Background()
+ slog.Warn("DEPRECATED: " + serviceName + " is deprecated and no longer has any functionality. " +
+ "This service is left in place until deployment code is updated accordingly.")
- cfg := &QueryConfig{}
+ // Start minimal HTTP server for health checks (required for cloud deployments)
+ mux := http.NewServeMux()
+ mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
- cfg.ControllerFunc = func() runner.Controller[queryrunner.Body] {
- que := query.New(cfg)
- res := result.New(cfg, &result.Services{
- Query: que,
- })
- sync := resultsync.New(cfg)
- resset := resultset.New(cfg, &resultset.Services{
- Result: res,
- Query: que,
- Sync: sync,
- })
-
- c := queryrunner.New(&queryrunner.Services{
- ResultSet: resset,
- })
-
- return &c
+ server := &http.Server{
+ Addr: ":8080",
+ Handler: mux,
+ ReadHeaderTimeout: time.Minute,
}
- server, err := runner.New(ctx, cfg)
- if err != nil {
- slog.Error(err.Error())
- os.Exit(1)
- }
+ go func() {
+ slog.Info("Starting health check server on port 8080")
+ if err := server.ListenAndServe(); err != http.ErrServerClosed {
+ slog.Error("Health check server error", "error", err)
+ }
+ }()
- server.Listen(ctx)
+ // Wait for shutdown signal
+ sigChan := make(chan os.Signal, 1)
+ signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+ sig := <-sigChan
+ slog.Info(serviceName+" received shutdown signal, exiting gracefully", "signal", sig.String())
+
+ // Gracefully shutdown the HTTP server
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := server.Shutdown(ctx); err != nil {
+ slog.Error("Error shutting down health check server", "error", err)
+ }
}
diff --git a/cmd/querySyncRunner/main.go b/cmd/querySyncRunner/main.go
index 1dfe3983..35230c05 100644
--- a/cmd/querySyncRunner/main.go
+++ b/cmd/querySyncRunner/main.go
@@ -1,66 +1,59 @@
-// **Listens For**: Body Containing Document ID
-//
-// **Action**.
-//
-// Find the queries with no dependencies that do not have a valid result for a document.
-//
-// The reasoning for only extracting these is to start the query extraction process only for necessary queries, and the query runner will manage updating downstream dependencies.
-//
-// There are 2 situations that cause a query to show up in this list:
-//
-// The query has no required dependencies.
-//
-// All required dependencies have a valid result.
-//
-// An event with the document id is pushed to the DOCTEXT queue.
+// DEPRECATED: This runner is deprecated and no longer has any functionality.
+// It is left in place until deployment code is updated accordingly.
+// See remove_query_plan.md for details on the query removal.
package main
import (
"context"
"log/slog"
+ "net/http"
"os"
+ "os/signal"
+ "syscall"
+ "time"
- querysyncrunner "queryorchestration/api/querySyncRunner"
- resultsync "queryorchestration/internal/query/result/sync"
- querysync "queryorchestration/internal/query/sync"
- "queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/build"
- queryc "queryorchestration/internal/serviceconfig/queue/query"
-
- _ "github.com/lib/pq"
)
-type QuerySyncConfig struct {
- runner.BaseConfig[querysyncrunner.Body]
- queryc.QueryConfig
-}
+const serviceName = "querySyncRunner"
func main() {
- // Print version information before any environment checks
- build.PrintVersionInfo("querySyncRunner")
+ // Print version information
+ build.PrintVersionInfo(serviceName)
- ctx := context.Background()
+ slog.Warn("DEPRECATED: " + serviceName + " is deprecated and no longer has any functionality. " +
+ "This service is left in place until deployment code is updated accordingly.")
- cfg := &QuerySyncConfig{}
+ // Start minimal HTTP server for health checks (required for cloud deployments)
+ mux := http.NewServeMux()
+ mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
- cfg.ControllerFunc = func() runner.Controller[querysyncrunner.Body] {
- sync := resultsync.New(cfg)
- svc := querysync.New(cfg, &querysync.Services{
- ResultSync: sync,
- })
-
- c := querysyncrunner.New(&querysyncrunner.Services{
- QuerySync: svc,
- })
-
- return &c
+ server := &http.Server{
+ Addr: ":8080",
+ Handler: mux,
+ ReadHeaderTimeout: time.Minute,
}
- server, err := runner.New(ctx, cfg)
- if err != nil {
- slog.Error(err.Error())
- os.Exit(1)
- }
+ go func() {
+ slog.Info("Starting health check server on port 8080")
+ if err := server.ListenAndServe(); err != http.ErrServerClosed {
+ slog.Error("Health check server error", "error", err)
+ }
+ }()
- server.Listen(ctx)
+ // Wait for shutdown signal
+ sigChan := make(chan os.Signal, 1)
+ signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+ sig := <-sigChan
+ slog.Info(serviceName+" received shutdown signal, exiting gracefully", "signal", sig.String())
+
+ // Gracefully shutdown the HTTP server
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := server.Shutdown(ctx); err != nil {
+ slog.Error("Error shutting down health check server", "error", err)
+ }
}
diff --git a/cmd/queryVersionSyncRunner/main.go b/cmd/queryVersionSyncRunner/main.go
index 0dbe19d9..b4f5f075 100644
--- a/cmd/queryVersionSyncRunner/main.go
+++ b/cmd/queryVersionSyncRunner/main.go
@@ -1,52 +1,59 @@
-// **Listens For**: Body Containing Query ID
-//
-// **Action**.
-//
-// Adds an event for each client into the CLIENTSYNC queue for the given query, i.e. each client that contains the query in its dependency tree.
+// DEPRECATED: This runner is deprecated and no longer has any functionality.
+// It is left in place until deployment code is updated accordingly.
+// See remove_query_plan.md for details on the query removal.
package main
import (
"context"
"log/slog"
+ "net/http"
"os"
+ "os/signal"
+ "syscall"
+ "time"
- queryversionsyncrunner "queryorchestration/api/queryVersionSyncRunner"
- queryversionsync "queryorchestration/internal/query/versionsync"
- "queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/build"
- "queryorchestration/internal/serviceconfig/queue/clientsync"
-
- _ "github.com/lib/pq"
)
-type QueryVersionSyncConfig struct {
- runner.BaseConfig[queryversionsyncrunner.Body]
- clientsync.ClientSyncConfig
-}
+const serviceName = "queryVersionSyncRunner"
func main() {
- // Print version information before any environment checks
- build.PrintVersionInfo("queryVersionSyncRunner")
+ // Print version information
+ build.PrintVersionInfo(serviceName)
- ctx := context.Background()
+ slog.Warn("DEPRECATED: " + serviceName + " is deprecated and no longer has any functionality. " +
+ "This service is left in place until deployment code is updated accordingly.")
- cfg := &QueryVersionSyncConfig{}
+ // Start minimal HTTP server for health checks (required for cloud deployments)
+ mux := http.NewServeMux()
+ mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ })
- cfg.ControllerFunc = func() runner.Controller[queryversionsyncrunner.Body] {
- svc := queryversionsync.New(cfg)
-
- c := queryversionsyncrunner.New(&queryversionsyncrunner.Services{
- Sync: svc,
- })
-
- return &c
+ server := &http.Server{
+ Addr: ":8080",
+ Handler: mux,
+ ReadHeaderTimeout: time.Minute,
}
- server, err := runner.New(ctx, cfg)
- if err != nil {
- slog.Error(err.Error())
- os.Exit(1)
- }
+ go func() {
+ slog.Info("Starting health check server on port 8080")
+ if err := server.ListenAndServe(); err != http.ErrServerClosed {
+ slog.Error("Health check server error", "error", err)
+ }
+ }()
- server.Listen(ctx)
+ // Wait for shutdown signal
+ sigChan := make(chan os.Signal, 1)
+ signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+ sig := <-sigChan
+ slog.Info(serviceName+" received shutdown signal, exiting gracefully", "signal", sig.String())
+
+ // Gracefully shutdown the HTTP server
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := server.Shutdown(ctx); err != nil {
+ slog.Error("Error shutting down health check server", "error", err)
+ }
}
diff --git a/cmd/textExtractor/main.go b/cmd/textExtractor/main.go
index 797c3cb5..379b5212 100644
--- a/cmd/textExtractor/main.go
+++ b/cmd/textExtractor/main.go
@@ -13,17 +13,17 @@ import (
"queryorchestration/internal/serviceconfig/aws"
"queryorchestration/internal/serviceconfig/build"
"queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/textract"
documenttext "queryorchestration/internal/document/text"
documenttypes "queryorchestration/internal/document/types"
)
+// Config provides the configuration for the text extractor tool.
+// Note: Query-related configuration has been removed. See remove_query_plan.md.
type Config struct {
serviceconfig.BaseConfig
textract.TextractConfig
- querysync.QuerySyncConfig
objectstore.ObjectStoreConfig
}
diff --git a/cmd/textGenerator_test/text/textdetection.go b/cmd/textGenerator_test/text/textdetection.go
index c6184d1e..b94566f7 100644
--- a/cmd/textGenerator_test/text/textdetection.go
+++ b/cmd/textGenerator_test/text/textdetection.go
@@ -9,20 +9,21 @@ import (
"io"
"log/slog"
"os"
+ "time"
+
documenttext "queryorchestration/internal/document/text"
documenttypes "queryorchestration/internal/document/types"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/aws"
"queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/textract"
- "time"
)
+// Config provides the configuration for the text generator test tool.
+// Note: Query-related configuration has been removed. See remove_query_plan.md.
type Config struct {
serviceconfig.BaseConfig
textract.TextractConfig
- querysync.QuerySyncConfig
objectstore.ObjectStoreConfig
}
diff --git a/cmd/textGenerator_test/textract/textdetection.go b/cmd/textGenerator_test/textract/textdetection.go
index 399c19f9..edd40748 100644
--- a/cmd/textGenerator_test/textract/textdetection.go
+++ b/cmd/textGenerator_test/textract/textdetection.go
@@ -9,23 +9,24 @@ import (
"fmt"
"log/slog"
"os"
+ "sync"
+ "time"
+
documenttext "queryorchestration/internal/document/text"
documenttypes "queryorchestration/internal/document/types"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/aws"
"queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/textract"
- "sync"
- "time"
awstextract "github.com/aws/aws-sdk-go-v2/service/textract"
)
+// Config provides the configuration for the text generator test tool.
+// Note: Query-related configuration has been removed. See remove_query_plan.md.
type Config struct {
serviceconfig.BaseConfig
textract.TextractConfig
- querysync.QuerySyncConfig
objectstore.ObjectStoreConfig
}
diff --git a/deployment_env_variables.md b/deployment_env_variables.md
index 94d3846a..df7121ad 100644
--- a/deployment_env_variables.md
+++ b/deployment_env_variables.md
@@ -24,10 +24,7 @@ These variables should be **IDENTICAL** for all services within the same environ
| `DOCUMENT_SYNC_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/document_sync` | Queue for document sync |
| `DOCUMENT_CLEAN_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/document_clean` | Queue for document cleaning |
| `DOCUMENT_TEXT_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/document_text` | Queue for text extraction |
-| `QUERY_SYNC_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/query_sync` | Queue for query sync |
-| `QUERY_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/query` | Queue for query execution |
| `CLIENT_SYNC_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/client_sync` | Queue for client sync |
-| `QUERY_VERSION_SYNC_URL` | `https://sqs.us-east-2.amazonaws.com/476114113988/query_version_sync` | Queue for query version sync |
### 3. AWS Configuration (SHARED across all services)
@@ -133,32 +130,11 @@ These should be **IDENTICAL** for all services:
- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/document_text`
**Also Needs:**
-- `QUERY_SYNC_URL`
- `BUCKET` (S3 bucket name)
- AWS config (shared)
- Database config (shared)
- Auth config (shared)
-### querySyncRunner
-**Unique Variables:**
-- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/query_sync`
-
-**Also Needs:**
-- `QUERY_URL`
-- AWS config (shared)
-- Database config (shared)
-- Auth config (shared)
-
-### queryRunner
-**Unique Variables:**
-- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/query`
-
-**Also Needs:**
-- `QUERY_URL` (same as QUEUE_URL - self-referential)
-- AWS config (shared)
-- Database config (shared)
-- Auth config (shared)
-
### clientSyncRunner
**Unique Variables:**
- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/client_sync`
@@ -169,22 +145,11 @@ These should be **IDENTICAL** for all services:
- Database config (shared)
- Auth config (shared)
-### queryVersionSyncRunner
-**Unique Variables:**
-- `QUEUE_URL` = `https://sqs.us-east-2.amazonaws.com/476114113988/query_version_sync`
-
-**Also Needs:**
-- `CLIENT_SYNC_URL`
-- AWS config (shared)
-- Database config (shared)
-- Auth config (shared)
-
### queryAPI (REST API Service)
**Does NOT need `QUEUE_URL`**
**Needs:**
- `CLIENT_SYNC_URL`
-- `QUERY_VERSION_SYNC_URL`
- `BUCKET` (S3 bucket name)
- `DEBUG` (optional, set to `true` for development)
- AWS config (shared)
diff --git a/docs/README.md b/docs/README.md
index 79e35fdc..06d016d0 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -56,7 +56,7 @@ If eligible, send an event to the DOCCLEAN queue.
If the document text is not already extracted according to the collector standards, trigger the textract job for text detection.
-Otherwise, add an event to the QUERYSYNC queue.
+This is the terminal step in the document processing pipeline. Query processing has been removed - pipeline ends after text extraction.
## healthcheck
@@ -86,49 +86,25 @@ The entities in question:
- Client - A client is a single client of the Doczy project, all configurations and clients for a client will be in reference to the respective client.
-- Collector - A collector specifies the required queries for export for a given client.
+- Collector - A collector specifies document processing settings for a given client.
- Sample - A sample is a subset of documents to be used for testing. When it is active a client will only process documents in the sample.
-- Query - A query is a block of processing of data which takes an input and produces an output.
-
- Export - An export is a manually started process which outputs the results for a client to a given location.
+Note: Query functionality has been removed from this API. See remove_query_plan.md for details.
+
## queryRunner
-**Listens For**: Body Containing Document ID and Query ID
-
-**Action**.
-
-Run the query for the given document and store the result.
-
-Events with the document id and downstream query ids are pushed to the QUERY queue.
+DEPRECATED: This runner is deprecated and no longer has any functionality. It is left in place until deployment code is updated accordingly. See remove_query_plan.md for details on the query removal.
## querySyncRunner
-**Listens For**: Body Containing Document ID
-
-**Action**.
-
-Find the queries with no dependencies that do not have a valid result for a document.
-
-The reasoning for only extracting these is to start the query extraction process only for necessary queries, and the query runner will manage updating downstream dependencies.
-
-There are 2 situations that cause a query to show up in this list:
-
-The query has no required dependencies.
-
-All required dependencies have a valid result.
-
-An event with the document id is pushed to the DOCTEXT queue.
+DEPRECATED: This runner is deprecated and no longer has any functionality. It is left in place until deployment code is updated accordingly. See remove_query_plan.md for details on the query removal.
## queryVersionSyncRunner
-**Listens For**: Body Containing Query ID
-
-**Action**.
-
-Adds an event for each client into the CLIENTSYNC queue for the given query, i.e. each client that contains the query in its dependency tree.
+DEPRECATED: This runner is deprecated and no longer has any functionality. It is left in place until deployment code is updated accordingly. See remove_query_plan.md for details on the query removal.
## storeEventRunner
@@ -144,7 +120,13 @@ It supports the document initialisation and document text processing stages. Eve
## cognito.auth.harness
-## permit.harness
+## permit.setup
+
+setup_permit.go - CLI tool to configure Permit.io roles and resources This tool reads a YAML configuration file and creates resources, actions, and roles in a Permit.io project/environment using the Permit.io REST API.
+
+Usage:
+
+
## user.creation.tool
@@ -160,4 +142,6 @@ cognito.go handles AWS Cognito User Pool operations for user identity management
permit.go handles Permit.io API operations for user management and role-based access control (RBAC). This file provides functions to create, delete, search, and manage users in Permit.io, as well as assign/unassign roles and validate role definitions. It integrates with Cognito user data to sync user information between the two systems.
+## permit.harness
+
Generated by Doczy
diff --git a/docs/ai.generated/01-system-overview.md b/docs/ai.generated/01-system-overview.md
index 225d8e81..11450002 100644
--- a/docs/ai.generated/01-system-overview.md
+++ b/docs/ai.generated/01-system-overview.md
@@ -1,16 +1,16 @@
# System Overview
-## DoczyAI Query Orchestration Platform
+## DoczyAI Document Processing Platform
### Purpose and Domain
-DoczyAI is a cloud-native document processing and query orchestration platform designed to extract structured information from documents (primarily PDFs) using configurable, hierarchical queries. The system serves as a multi-tenant platform where organizations can upload documents, define custom data extraction rules, and receive structured results for business intelligence and analysis.
+DoczyAI is a cloud-native document processing platform designed to extract structured information from documents (primarily PDFs). The system serves as a multi-tenant platform where organizations can upload documents, perform text extraction, and organize documents with folders and labels for business intelligence and analysis.
### Core Business Capabilities
- **Multi-tenant Document Processing**: Isolated processing environments per client
- **Batch Document Processing**: ZIP archive uploads for processing multiple documents simultaneously
-- **Configurable Query System**: Custom data extraction rules with dependency management
+- **Text Extraction Pipeline**: Automated OCR and text extraction from documents
- **Automated Processing Pipeline**: Seamless flow from document upload to structured output
-- **Version Management**: Complete versioning for queries and configurations
+- **Version Management**: Complete versioning for configurations
- **Export and Integration**: Structured data export capabilities for downstream systems
- **Enhanced Observability**: Comprehensive metrics and monitoring with Prometheus integration
@@ -30,7 +30,7 @@ DoczyAI is a cloud-native document processing and query orchestration platform d
### System Architecture Pattern
The system follows an **event-driven microservices architecture** with the following characteristics:
-- **12 distinct services**: 1 HTTP API + 8 queue-based runners + 3 utility services
+- **8 distinct services**: 1 HTTP API + 6 queue-based runners + 1 utility service
- **Asynchronous communication**: SQS queues between services with batch processing capabilities
- **Background Task Framework**: Comprehensive background task execution with periodic scheduling
- **State management**: PostgreSQL as central data store with enhanced schema for batch operations
@@ -56,13 +56,10 @@ graph TB
DI --> DS[docSyncRunner
8083]
DS --> DC[docCleanRunner
8084]
DC --> DT[docTextRunner
8085]
- DT --> QS[querySyncRunner
8087]
- QS --> QR[queryRunner
8088]
end
-
+
subgraph "Synchronization Services"
CS[clientSyncRunner
8089]
- QVS[queryVersionSyncRunner
8090]
end
subgraph "Infrastructure Services"
@@ -87,11 +84,8 @@ graph TB
DS --> DB
DC --> S3
DT --> TEXTRACT
- QS --> DB
- QR --> DB
-
+
CS --> SQS
- QVS --> SQS
All_Services --> PROM
```
@@ -125,7 +119,7 @@ graph TB
2. **AWS SQS**: Inter-service messaging and event processing
3. **AWS Textract**: OCR and text extraction from documents
4. **AWS Cognito**: User authentication and session management
-5. **PostgreSQL**: Central data persistence and query processing
+5. **PostgreSQL**: Central data persistence
### API Integration
- **REST API**: OpenAPI 3.0.3 specification with Swagger UI
diff --git a/docs/ai.generated/02-service-architecture.md b/docs/ai.generated/02-service-architecture.md
index fab915e9..b2290739 100644
--- a/docs/ai.generated/02-service-architecture.md
+++ b/docs/ai.generated/02-service-architecture.md
@@ -12,7 +12,8 @@
- Client management and authentication
- Document upload and metadata management (single and batch)
- Batch upload processing with ZIP archive support
- - Query definition and testing
+ - Folder and label management
+ - Field extraction configuration
- Export operations and status monitoring
- Background task coordination and monitoring
- User interface and API documentation
@@ -57,25 +58,9 @@
- **Port**: 8085
- **Type**: Queue-based processor
- **Input**: `{document_id}` from DOCTEXT queue
-- **Output**: Events to QUERYSYNC queue
+- **Output**: Text stored in database, processing complete
- **Dependencies**: AWS Textract, S3 ObjectStore
-#### querySyncRunner - Query Dependency Resolution
-- **Purpose**: Identifies executable queries for documents
-- **Port**: 8087
-- **Type**: Queue-based processor
-- **Input**: `{document_id}` from QUERYSYNC queue
-- **Output**: Events to QUERY queue
-- **Logic**: Finds queries with satisfied dependencies
-
-#### queryRunner - Query Execution Engine
-- **Purpose**: Executes queries and stores results
-- **Port**: 8088
-- **Type**: Queue-based processor
-- **Input**: `{document_id, query_id}` from QUERY queue
-- **Output**: Events to downstream QUERY queue
-- **Responsibilities**: Query execution and result persistence
-
### Synchronization Services
#### clientSyncRunner - Client-Level Synchronization
@@ -85,13 +70,6 @@
- **Input**: `{client_id}` from CLIENTSYNC queue
- **Output**: Events to DOCSYNC queue for each client document
-#### queryVersionSyncRunner - Query Change Propagation
-- **Purpose**: Propagates query updates to affected clients
-- **Port**: 8090
-- **Type**: Queue-based processor
-- **Input**: `{query_id}` from QUERYVERSIONSYNC queue
-- **Output**: Events to CLIENTSYNC queue for affected clients
-
### Utility Services
#### healthcheck - Health Monitoring
@@ -134,10 +112,8 @@ sequenceDiagram
participant Sync as docSyncRunner
participant Clean as docCleanRunner
participant Text as docTextRunner
- participant QSync as querySyncRunner
- participant Query as queryRunner
participant DB as Database
-
+
Client->>S3: Upload Document
S3->>Store: S3 Event Notification
Store->>Init: DOCINIT Queue
@@ -149,29 +125,22 @@ sequenceDiagram
Clean->>Text: DOCTEXT Queue
Text->>S3: Read Document
Text->>AWS: Textract Processing
- Text->>QSync: QUERYSYNC Queue
- QSync->>DB: Find Executable Queries
- QSync->>Query: QUERY Queue
- Query->>DB: Execute & Store Results
+ Text->>DB: Store Extracted Text
```
-### Query Version Management Flow
-
-[link to rendered version here](https://mermaid.live/edit#pako:eNp9kctqwzAQRX9l0KqF9ge8CBTZC0Nx_GgMgW6ENG4GLFmVZEII-ffKj_SVUi2EGJ0z3GHOTA4KWQLM4_uIRmJK4s0J_WogHitcIElWmABPSpP5o1zmIDxE2Z3i-xao2uYTaNF5GkxzMrIejUF3i_OZlj2hCf9xJVnsyeBEp4McNX4rLvhyz7kfN5sYLoGdVSIgVFMYSLEjQyEGWskyj1zMm0C1y-p9m9VNvi2afcEnY1zbRiBiPFL8Oc-Kl69_uLPoQHQdyoBqHeJ-sfgkXfMlkG75b2_BQa3DrN5V-WHXaN0g0Xs4UjhAgcdlJPYATKPTglTc6ZmFA-p5uwo7MfaBXS4faYamhw==)
+### Client Synchronization Flow
```mermaid
sequenceDiagram
participant Admin
participant API as queryAPI
- participant QVS as queryVersionSyncRunner
participant CS as clientSyncRunner
participant Pipeline as Document Pipeline
-
- Admin->>API: Update Query Definition
- API->>QVS: QUERYVERSIONSYNC Queue
- QVS->>CS: CLIENTSYNC Queue (per affected client)
+
+ Admin->>API: Update Client Configuration
+ API->>CS: CLIENTSYNC Queue
CS->>Pipeline: DOCSYNC Queue (per client document)
- Pipeline->>Pipeline: Reprocess with New Query
+ Pipeline->>Pipeline: Reprocess Documents
```
## Communication Patterns
@@ -191,10 +160,7 @@ All services communicate through **AWS SQS queues** with the following patterns:
| DOCSYNC | docInitRunner, clientSyncRunner | docSyncRunner | `{document_id}` |
| DOCCLEAN | docSyncRunner | docCleanRunner | `{document_id}` |
| DOCTEXT | docCleanRunner | docTextRunner | `{document_id}` |
-| QUERYSYNC | docTextRunner | querySyncRunner | `{document_id}` |
-| QUERY | querySyncRunner, queryRunner | queryRunner | `{document_id, query_id}` |
-| CLIENTSYNC | queryVersionSyncRunner | clientSyncRunner | `{client_id}` |
-| QUERYVERSIONSYNC | queryAPI | queryVersionSyncRunner | `{query_id}` |
+| CLIENTSYNC | queryAPI | clientSyncRunner | `{client_id}` |
| BATCH_PROCESSING | queryAPI (batch worker) | Internal batch processor | `{batch_id, files[]}` |
### Database Access Patterns
@@ -208,7 +174,7 @@ All services communicate through **AWS SQS queues** with the following patterns:
### Horizontal Scaling Strategy
- **API Service**: Load balancer with multiple instances
- **Queue Processors**: Auto-scaling based on queue depth
-- **Database**: Read replicas for query scaling
+- **Database**: Read replicas for scaling
- **Storage**: S3 scales automatically
### Performance Optimizations
diff --git a/docs/ai.generated/README.md b/docs/ai.generated/README.md
index aed6c649..6ff3b7c6 100644
--- a/docs/ai.generated/README.md
+++ b/docs/ai.generated/README.md
@@ -1,8 +1,8 @@
-# DoczyAI Query Orchestration Platform Documentation
+# DoczyAI Document Processing Platform Documentation
## Overview
-This documentation provides comprehensive coverage of the DoczyAI Query Orchestration Platform - a cloud-native document processing and query orchestration system designed to extract structured information from documents using configurable, hierarchical queries.
+This documentation provides comprehensive coverage of the DoczyAI Document Processing Platform - a cloud-native document processing system designed to extract structured information from documents.
## Documentation Structure
@@ -15,7 +15,7 @@ This documentation provides comprehensive coverage of the DoczyAI Query Orchestr
- Deployment architecture and scalability
2. **[Service Architecture](02-service-architecture.md)**
- - Detailed service catalog (12 services)
+ - Detailed service catalog (8 services)
- Document processing pipeline flow
- Inter-service communication patterns
- Queue architecture and message formats
@@ -28,8 +28,8 @@ This documentation provides comprehensive coverage of the DoczyAI Query Orchestr
4. **[Data Architecture](04-data-architecture.md)**
- Database schema and entity relationships
- - Versioning system and temporal queries
- - Query dependency management
+ - Versioning system
+ - Document organization (folders and labels)
- Performance optimization strategies
### 🛠️ Developer Documentation
@@ -82,9 +82,9 @@ This documentation provides comprehensive coverage of the DoczyAI Query Orchestr
### 🏗️ Architecture
- **Event-driven microservices** with SQS-based communication
-- **12 distinct services**: 1 HTTP API + 8 runners + 3 utilities
+- **8 distinct services**: 1 HTTP API + 6 runners + 1 utility
- **Multi-tenant design** with client isolation
-- **Comprehensive versioning** for queries and configurations
+- **Comprehensive versioning** for configurations
### 🔧 Technology Stack
- **Backend**: Go 1.24.0 with Echo framework
@@ -95,10 +95,10 @@ This documentation provides comprehensive coverage of the DoczyAI Query Orchestr
### 📊 Key Capabilities
- **Document Processing**: PDF upload, cleaning, and text extraction
-- **Query Orchestration**: Configurable data extraction with dependencies
+- **Text Extraction**: Automated OCR with AWS Textract
- **Multi-tenant Support**: Isolated client environments
+- **Document Organization**: Folders and labels for document management
- **Export Operations**: Structured data export in multiple formats
-- **Version Management**: Full query and configuration versioning
## Documentation Tools
diff --git a/docs/ai.generated/query-system-explanation.md b/docs/ai.generated/query-system-explanation.md
deleted file mode 100644
index 21a98185..00000000
--- a/docs/ai.generated/query-system-explanation.md
+++ /dev/null
@@ -1,140 +0,0 @@
-# Query System Explanation
-
-## Overview
-
-The query system in this application is designed to extract structured data from processed documents. Queries are configurable data extraction rules that can depend on each other, creating a pipeline of data transformations.
-
-## Query Types
-
-The system currently supports two query types:
-
-### 1. CONTEXT_FULL Query
-- **Purpose**: Originally intended to provide the full document text/context
-- **Current Implementation**: Returns a hardcoded JSON placeholder `{"keyone":"valueone","keytwo":"valuetwo"}`
-- **No Dependencies**: This query type doesn't require any other queries to run first
-- **Note**: The current implementation appears to be a mock/placeholder - in production, this would likely return the actual extracted text from the document
-
-### 2. JSON_EXTRACTOR Query
-- **Purpose**: Extracts specific values from JSON data using path expressions
-- **Dependencies**: Requires exactly one other query result as input (typically a CONTEXT_FULL query)
-- **Configuration**: Takes a JSON config with a `path` field specifying what to extract
-- **Uses gjson**: Leverages the `tidwall/gjson` library for JSON path extraction
-
-## How Queries Work
-
-### Query Definition Structure
-```go
-type Query {
- ID uuid
- Type queryType // CONTEXT_FULL or JSON_EXTRACTOR
- ActiveVersion int32 // Current version being used
- RequiredQueryIDs []uuid // Dependencies - other queries that must run first
- Config string // JSON configuration (e.g., {"path":"keyone"})
-}
-```
-
-### Example Query Chain
-
-1. **First Query (CONTEXT_FULL)**:
- - Type: `CONTEXT_FULL`
- - Config: None needed
- - Dependencies: None
- - Returns: `{"keyone":"valueone","keytwo":"valuetwo"}`
-
-2. **Second Query (JSON_EXTRACTOR)**:
- - Type: `JSON_EXTRACTOR`
- - Config: `{"path":"keyone"}`
- - Dependencies: [First Query ID]
- - Input: Result from First Query
- - Returns: `"valueone"`
-
-3. **Third Query (JSON_EXTRACTOR)** - could extract from second:
- - Type: `JSON_EXTRACTOR`
- - Config: `{"path":"some.nested.path"}`
- - Dependencies: [Second Query ID]
- - Would extract from the result of the second query
-
-## Query Execution Flow
-
-### 1. querySyncRunner Phase
-- After document text extraction completes, querySyncRunner is triggered
-- It queries the database for all queries that:
- - Are configured for this client (via collector configuration)
- - Have NO dependencies, OR
- - Have all dependencies already satisfied (results exist in database)
-- For each eligible query, sends a message to the QUERY queue
-
-### 2. queryRunner Phase
-- Receives {document_id, query_id} from QUERY queue
-- Retrieves the query definition and active version
-- If query has dependencies:
- - Fetches results from all required queries
- - Validates all dependencies are satisfied
-- Executes the appropriate processor based on query type:
- - CONTEXT_FULL: Returns the mock JSON (should return document text)
- - JSON_EXTRACTOR: Extracts specified path from dependency result
-- Stores the result in database with version tracking
-- **Triggers dependent queries**: Finds all queries that depend on this one and sends them to QUERY queue
-
-### 3. Recursive Processing
-- As each query completes, it triggers its dependents
-- This creates a cascade effect through the dependency graph
-- Continues until all queries in the chain have been executed
-
-## Collector Configuration
-
-Collectors define which queries should run for each client:
-
-```sql
--- collectorQueries table maps queries to clients
-CREATE TABLE collectorQueries (
- clientId varchar(255), -- Which client
- name varchar(255), -- Named reference (e.g., "JSON_QUERY")
- queryId uuid, -- Which query to run
- ...
-)
-```
-
-When documents are processed, only queries configured in the client's collector will be executed.
-
-## Current Implementation Status
-
-### What's Working
-- Query dependency resolution and scheduling
-- Recursive execution of dependent queries
-- Version tracking for queries and results
-- JSON path extraction from query results
-- Client-specific query configuration via collectors
-
-### What Appears Incomplete
-- **CONTEXT_FULL Implementation**: Currently returns mock data instead of actual document text
-- The system stores extracted text in S3 (via docTextRunner) but CONTEXT_FULL doesn't retrieve it
-- There's infrastructure for text storage (`documentTextExtractions` table) but it's not connected to the query processor
-
-## Real-World Use Case
-
-In a production scenario, this system would work like:
-
-1. **Document uploaded** → Text extracted via OCR
-2. **Base Query** (CONTEXT_FULL) → Returns full document text as JSON
-3. **Field Extraction Queries** (JSON_EXTRACTOR) → Extract specific fields like:
- - Customer name
- - Invoice number
- - Date
- - Amount
-4. **Derived Queries** → Further process extracted data:
- - Calculate totals
- - Validate formats
- - Cross-reference with other data
-
-## Key Design Benefits
-
-1. **Flexibility**: New extraction rules can be added without code changes
-2. **Reusability**: Queries can be shared across clients or documents
-3. **Dependency Management**: Complex multi-step extractions are handled automatically
-4. **Versioning**: Changes to queries don't break existing results
-5. **Client Isolation**: Each client can have different extraction rules for the same document types
-
-## Summary
-
-The query system is a sophisticated, dependency-aware data extraction pipeline. While the current implementation has placeholder logic for the base CONTEXT_FULL query, the infrastructure for chaining queries, managing dependencies, and extracting structured data from documents is fully functional. The system allows for complex, multi-step data extraction workflows to be defined declaratively and executed automatically as documents are processed.
\ No newline at end of file
diff --git a/go.mod b/go.mod
index 403fe9e3..8c7d0f31 100644
--- a/go.mod
+++ b/go.mod
@@ -30,10 +30,8 @@ require (
github.com/stretchr/testify v1.10.0
github.com/swaggo/echo-swagger v1.4.1
github.com/testcontainers/testcontainers-go v0.35.0
- github.com/tidwall/gjson v1.18.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0
go.opentelemetry.io/otel/sdk v1.34.0
- golang.org/x/sync v0.12.0
golang.org/x/time v0.11.0
)
@@ -112,8 +110,6 @@ require (
github.com/subosito/gotenv v1.6.0 // indirect
github.com/swaggo/files/v2 v2.0.2 // indirect
github.com/swaggo/swag v1.16.4 // indirect
- github.com/tidwall/match v1.1.1 // indirect
- github.com/tidwall/pretty v1.2.1 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
github.com/vektra/mockery/v2 v2.52.3 // indirect
@@ -128,6 +124,7 @@ require (
golang.org/x/image v0.25.0 // indirect
golang.org/x/mod v0.23.0 // indirect
golang.org/x/oauth2 v0.27.0 // indirect
+ golang.org/x/sync v0.12.0 // indirect
golang.org/x/term v0.30.0 // indirect
golang.org/x/tools v0.30.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect
diff --git a/go.sum b/go.sum
index f005a5ed..65d2d626 100644
--- a/go.sum
+++ b/go.sum
@@ -343,13 +343,6 @@ github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A=
github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg=
github.com/testcontainers/testcontainers-go v0.35.0 h1:uADsZpTKFAtp8SLK+hMwSaa+X+JiERHtd4sQAFmXeMo=
github.com/testcontainers/testcontainers-go v0.35.0/go.mod h1:oEVBj5zrfJTrgjwONs1SsRbnBtH9OKl+IGl3UMcr2B4=
-github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
-github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
-github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
-github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
-github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU=
github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY=
github.com/tklauser/numcpus v0.9.0 h1:lmyCHtANi8aRUgkckBgoDk1nHCux3n2cgkJLXdQGPDo=
diff --git a/internal/collector/get.go b/internal/collector/get.go
index fda9dae9..437f92c2 100644
--- a/internal/collector/get.go
+++ b/internal/collector/get.go
@@ -2,10 +2,9 @@ package collector
import (
"context"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
)
+// GetByClientID retrieves a client's collector configuration by client ID.
func (s *Service) GetByClientID(ctx context.Context, clientID string) (*Collector, error) {
dbColl, err := s.cfg.GetDBQueries().GetCollectorByClientID(ctx, clientID)
if err != nil {
@@ -14,22 +13,3 @@ func (s *Service) GetByClientID(ctx context.Context, clientID string) (*Collecto
return parseDBCollector(dbColl)
}
-
-func (s *Service) ListQueries(ctx context.Context, id string) ([]*resultprocessor.Query, error) {
- queries, err := s.cfg.GetDBQueries().ListCollectorQueries(ctx, id)
- if err != nil {
- return nil, err
- }
-
- cleanQueries := make([]*resultprocessor.Query, len(queries))
- for index, dbQuery := range queries {
- cleanQuery, err := resultprocessor.ParseDBCollectorQuery(dbQuery)
- if err != nil {
- return nil, err
- }
-
- cleanQueries[index] = cleanQuery
- }
-
- return cleanQueries, nil
-}
diff --git a/internal/collector/get_test.go b/internal/collector/get_test.go
index 7e187f88..41b5460e 100644
--- a/internal/collector/get_test.go
+++ b/internal/collector/get_test.go
@@ -1,15 +1,12 @@
package collector_test
import (
- "fmt"
"testing"
"queryorchestration/internal/collector"
"queryorchestration/internal/database/repository"
- resultprocessor "queryorchestration/internal/query/result/processor"
"queryorchestration/internal/serviceconfig"
- "github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -32,55 +29,15 @@ func TestGetByClientID(t *testing.T) {
ClientID: "hi",
MinCleanVersion: minCleanV,
MinTextVersion: minTextV,
- Fields: map[string]uuid.UUID{
- "example_key": uuid.New(),
- },
}
pool.ExpectQuery("name: GetCollectorByClientID :one").WithArgs(ogc.ClientID).
WillReturnRows(
- pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
- AddRow(ogc.ClientID, ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))),
+ pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion"}).
+ AddRow(ogc.ClientID, ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion),
)
coll, err := svc.GetByClientID(ctx, ogc.ClientID)
require.NoError(t, err)
assert.EqualExportedValues(t, ogc, *coll)
}
-
-func TestListQueries(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
-
- svc := collector.New(cfg)
-
- clientId := "id"
- ogc := []*resultprocessor.Query{
- {
- ID: uuid.New(),
- Type: resultprocessor.TypeContextFull,
- Version: 2,
- },
- }
-
- v := int32(2)
- ty := repository.NullQuerytype{
- Querytype: repository.QuerytypeContextFull,
- Valid: true,
- }
- pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(clientId).
- WillReturnRows(
- pgxmock.NewRows([]string{"clientId", "queryId", "type", "queryVersion", "requiredIds"}).
- AddRow(&clientId, &ogc[0].ID, ty, &v, nil),
- )
-
- qs, err := svc.ListQueries(ctx, clientId)
- require.NoError(t, err)
- assert.Len(t, qs, 1)
- assert.EqualExportedValues(t, *ogc[0], *qs[0])
-}
diff --git a/internal/collector/parse.go b/internal/collector/parse.go
index 6ae69715..40234586 100644
--- a/internal/collector/parse.go
+++ b/internal/collector/parse.go
@@ -1,36 +1,21 @@
package collector
import (
- "encoding/json"
- "fmt"
-
"queryorchestration/internal/database/repository"
-
- "github.com/google/uuid"
)
+// parseDBCollector converts a database collector record to a Collector struct.
+// Query fields have been removed - only document processing metadata is returned.
func parseDBCollector(c *repository.Fullactivecollector) (*Collector, error) {
if c == nil {
return nil, nil
}
- var fields map[string]uuid.UUID
-
- if len(c.Fields) > 0 {
- err := json.Unmarshal(c.Fields, &fields)
- if err != nil {
- return nil, fmt.Errorf("error unmarshalling: %w", err)
- }
- } else {
- fields = map[string]uuid.UUID{}
- }
-
return &Collector{
ClientID: c.Clientid,
MinCleanVersion: c.Mincleanversion,
MinTextVersion: c.Mintextversion,
ActiveVersion: c.Activeversion,
LatestVersion: c.Latestversion,
- Fields: fields,
}, nil
}
diff --git a/internal/collector/parse_test.go b/internal/collector/parse_test.go
index 2b6ade56..ee6ff664 100644
--- a/internal/collector/parse_test.go
+++ b/internal/collector/parse_test.go
@@ -1,12 +1,10 @@
package collector
import (
- "fmt"
"testing"
"queryorchestration/internal/database/repository"
- "github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -22,15 +20,11 @@ func TestParseDBCollector(t *testing.T) {
ClientID: "hi",
MinCleanVersion: minCleanV,
MinTextVersion: minTextV,
- Fields: map[string]uuid.UUID{
- "example_key": uuid.New(),
- },
}
c, err = parseDBCollector(&repository.Fullactivecollector{
Clientid: ogc.ClientID,
Mincleanversion: ogc.MinCleanVersion,
Mintextversion: ogc.MinTextVersion,
- Fields: []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"])),
})
require.NoError(t, err)
assert.EqualExportedValues(t, ogc, *c)
@@ -39,7 +33,6 @@ func TestParseDBCollector(t *testing.T) {
ogc.MinTextVersion = 0
c, err = parseDBCollector(&repository.Fullactivecollector{
Clientid: ogc.ClientID,
- Fields: []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"])),
})
require.NoError(t, err)
assert.EqualExportedValues(t, ogc, *c)
diff --git a/internal/collector/service.go b/internal/collector/service.go
index 5ba33936..83d74846 100644
--- a/internal/collector/service.go
+++ b/internal/collector/service.go
@@ -2,17 +2,16 @@ package collector
import (
"queryorchestration/internal/serviceconfig"
-
- "github.com/google/uuid"
)
+// Collector represents a client's document collection configuration.
+// Query processing has been removed - only document processing metadata remains.
type Collector struct {
ClientID string
MinCleanVersion int64
MinTextVersion int64
ActiveVersion int32
LatestVersion int32
- Fields map[string]uuid.UUID
}
type Service struct {
diff --git a/internal/collector/set/set.go b/internal/collector/set/set.go
index f5c19cad..da632ff4 100644
--- a/internal/collector/set/set.go
+++ b/internal/collector/set/set.go
@@ -11,15 +11,14 @@ import (
"queryorchestration/internal/serviceconfig/build"
"queryorchestration/internal/serviceconfig/queue"
"queryorchestration/internal/validation"
-
- "github.com/google/uuid"
)
+// SetParams defines parameters for updating a collector's configuration.
+// Query fields have been removed - only document processing settings remain.
type SetParams struct {
ActiveVersion *int32
MinCleanVersion *int64
MinTextVersion *int64
- Fields *map[string]uuid.UUID
}
func (s *Service) SetByClientId(ctx context.Context, id string, params *SetParams) error {
@@ -28,12 +27,12 @@ func (s *Service) SetByClientId(ctx context.Context, id string, params *SetParam
return err
}
- dbparams, err := s.getSetParams(ctx, id, current, params)
+ dbparams, err := s.getSetParams(id, current, params)
if err != nil {
return err
}
- err = s.submitSet(ctx, current, dbparams)
+ err = s.submitSet(ctx, dbparams)
if err != nil {
return err
}
@@ -64,20 +63,14 @@ type dbSetParams struct {
ActiveVersion *int32
MinCleanVersion *int64
MinTextVersion *int64
- Fields *map[string]uuid.UUID
}
-func (s *Service) getSetParams(ctx context.Context, id string, current *collector.Collector, params *SetParams) (*dbSetParams, error) {
+func (s *Service) getSetParams(id string, current *collector.Collector, params *SetParams) (*dbSetParams, error) {
err := s.normalizeCodeVersions(current, params)
if err != nil {
return nil, err
}
- fields, err := s.normalizeSetFieldsToDB(ctx, current.Fields, params.Fields)
- if err != nil {
- return nil, err
- }
-
err = s.normalizeActiveVersion(current, params)
if err != nil {
return nil, err
@@ -87,14 +80,9 @@ func (s *Service) getSetParams(ctx context.Context, id string, current *collecto
if err != nil {
return nil, err
}
- fieldsName, err := validation.GetFieldName(params, params.Fields)
- if err != nil {
- return nil, err
- }
if (params.ActiveVersion == nil || *params.ActiveVersion == current.LatestVersion+1) &&
- validation.AreAllPointersNilExcept(params, activeVersionName, fieldsName) &&
- fields == nil {
+ validation.AreAllPointersNilExcept(params, activeVersionName) {
return nil, errors.New("no changes")
}
@@ -103,7 +91,6 @@ func (s *Service) getSetParams(ctx context.Context, id string, current *collecto
ActiveVersion: params.ActiveVersion,
MinCleanVersion: params.MinCleanVersion,
MinTextVersion: params.MinTextVersion,
- Fields: fields,
}, nil
}
@@ -158,29 +145,7 @@ func (s *Service) normalizeActiveVersion(current *collector.Collector, params *S
return nil
}
-func (s *Service) normalizeSetFieldsToDB(ctx context.Context, current map[string]uuid.UUID, ofields *map[string]uuid.UUID) (*map[string]uuid.UUID, error) {
- if ofields == nil || *ofields == nil {
- return nil, nil
- } else if len(*ofields) == 0 {
- return nil, nil
- }
-
- dbm := map[string]uuid.UUID{}
- for name, id := range *ofields {
- dbm[name] = id
- }
-
- removeIDs := getRemoveFields(current, &dbm)
- addIDs := getAddFields(current, &dbm)
-
- if len(addIDs) == 0 && len(removeIDs) == 0 {
- return nil, nil
- }
-
- return s.normalizeFieldsToDB(ctx, ofields)
-}
-
-func (s *Service) submitSet(ctx context.Context, current *collector.Collector, params *dbSetParams) error {
+func (s *Service) submitSet(ctx context.Context, params *dbSetParams) error {
err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error {
activeName, err := validation.GetFieldName(params, params.ActiveVersion)
if err != nil {
@@ -228,33 +193,6 @@ func (s *Service) submitSet(ctx context.Context, current *collector.Collector, p
}
}
- if params.Fields != nil {
- removeIDs := getRemoveFields(current.Fields, params.Fields)
- for _, field := range removeIDs {
- err := qtx.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
- Clientid: params.ClientID,
- Queryid: field,
- Removedversion: &latestVersion,
- })
- if err != nil {
- return err
- }
- }
-
- addIDs := getAddFields(current.Fields, params.Fields)
- for key, field := range addIDs {
- err := qtx.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
- Clientid: params.ClientID,
- Name: key,
- Queryid: field,
- Addedversion: latestVersion,
- })
- if err != nil {
- return err
- }
- }
- }
-
slog.Debug("client collector updated", "update", *params)
return nil
@@ -265,73 +203,3 @@ func (s *Service) submitSet(ctx context.Context, current *collector.Collector, p
return nil
}
-
-func getRemoveFields(current map[string]uuid.UUID, update *map[string]uuid.UUID) []uuid.UUID {
- diff := []uuid.UUID{}
- if update == nil {
- return diff
- }
-
- for ckey, cid := range current {
- found := false
- for ukey, uid := range *update {
- if cid == uid &&
- ckey == ukey {
- found = true
- }
- }
- if !found {
- diff = append(diff, cid)
- }
- }
-
- return diff
-}
-
-func getAddFields(current map[string]uuid.UUID, update *map[string]uuid.UUID) map[string]uuid.UUID {
- diff := map[string]uuid.UUID{}
- if update == nil {
- return diff
- }
-
- for ukey, uid := range *update {
- if current[ukey] != uid {
- diff[ukey] = uid
- }
- }
-
- return diff
-}
-
-func (s *Service) normalizeFieldsToDB(ctx context.Context, ofields *map[string]uuid.UUID) (*map[string]uuid.UUID, error) {
- if ofields == nil || *ofields == nil {
- return nil, nil
- } else if len(*ofields) == 0 {
- return nil, nil
- }
-
- dbm := map[string]uuid.UUID{}
- for name, id := range *ofields {
- dbm[name] = id
- }
-
- dbids := []uuid.UUID{}
- for _, id := range dbm {
- dbids = append(dbids, id)
- }
-
- dedup := validation.DeduplicateArray(dbids)
-
- if len(dedup) != len(dbids) {
- return nil, errors.New("duplicate output fields")
- }
-
- exist, err := s.cfg.GetDBQueries().AllQueriesExist(ctx, dbids)
- if err != nil {
- return nil, err
- } else if !exist {
- return nil, errors.New("not all required ids are present")
- }
-
- return &dbm, nil
-}
diff --git a/internal/collector/set/set_test.go b/internal/collector/set/set_test.go
index a57d9173..34842582 100644
--- a/internal/collector/set/set_test.go
+++ b/internal/collector/set/set_test.go
@@ -54,8 +54,8 @@ func TestSet(t *testing.T) {
pool.ExpectQuery("name: GetCollectorByClientID :one").WithArgs(current.ClientID).
WillReturnRows(
- pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
- AddRow(current.ClientID, current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")),
+ pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion"}).
+ AddRow(current.ClientID, current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion),
)
pool.ExpectBeginTx(pgx.TxOptions{})
rv := current.LatestVersion + 1
@@ -95,8 +95,8 @@ func TestSet(t *testing.T) {
pool.ExpectQuery("name: GetCollectorByClientID :one").WithArgs(current.ClientID).
WillReturnRows(
- pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
- AddRow(current.ClientID, current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")),
+ pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion"}).
+ AddRow(current.ClientID, current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion),
)
pool.ExpectBeginTx(pgx.TxOptions{})
rv := current.LatestVersion + 1
diff --git a/internal/collector/set/setprivate_test.go b/internal/collector/set/setprivate_test.go
index 4232289a..f28bfd04 100644
--- a/internal/collector/set/setprivate_test.go
+++ b/internal/collector/set/setprivate_test.go
@@ -11,7 +11,6 @@ import (
queuemock "queryorchestration/mocks/queue"
"github.com/aws/aws-sdk-go-v2/service/sqs"
- "github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
@@ -26,8 +25,6 @@ type CollectorSetConfig struct {
func TestGetSetParams(t *testing.T) {
t.Run("all params", func(t *testing.T) {
- ctx := t.Context()
-
pool, err := pgxmock.NewPool()
require.NoError(t, err)
cfg := &CollectorSetConfig{}
@@ -52,39 +49,21 @@ func TestGetSetParams(t *testing.T) {
ActiveVersion: &aV,
MinCleanVersion: &minCleanV,
MinTextVersion: &minTextV,
- Fields: &map[string]uuid.UUID{
- "example_key": uuid.New(),
- },
}
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs([]uuid.UUID{(*params.Fields)["example_key"]}).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(true),
- )
-
clientId := "string"
- dbparams, err := svc.getSetParams(ctx, clientId, ¤t, ¶ms)
+ dbparams, err := svc.getSetParams(clientId, ¤t, ¶ms)
require.NoError(t, err)
assert.EqualExportedValues(t, &dbSetParams{
ClientID: clientId,
ActiveVersion: &aV,
MinCleanVersion: &minCleanV,
MinTextVersion: &minTextV,
- Fields: &map[string]uuid.UUID{
- "example_key": (*params.Fields)["example_key"],
- },
}, dbparams)
-
- (*params.Fields)["second_key"] = (*params.Fields)["example_key"]
- assert.Len(t, *params.Fields, 2)
- _, err = svc.getSetParams(ctx, clientId, ¤t, ¶ms)
- assert.Error(t, err)
})
t.Run("no params", func(t *testing.T) {
- ctx := t.Context()
-
pool, err := pgxmock.NewPool()
require.NoError(t, err)
cfg := &CollectorSetConfig{}
@@ -98,13 +77,11 @@ func TestGetSetParams(t *testing.T) {
current := collector.Collector{}
params := SetParams{}
- _, err = svc.getSetParams(ctx, current.ClientID, ¤t, ¶ms)
+ _, err = svc.getSetParams(current.ClientID, ¤t, ¶ms)
assert.EqualError(t, err, "no changes")
})
t.Run("ONLY active version to next latest", func(t *testing.T) {
- ctx := t.Context()
-
pool, err := pgxmock.NewPool()
require.NoError(t, err)
cfg := &CollectorSetConfig{}
@@ -124,42 +101,13 @@ func TestGetSetParams(t *testing.T) {
params := SetParams{
ActiveVersion: &version,
}
- _, err = svc.getSetParams(ctx, current.ClientID, ¤t, ¶ms)
- assert.EqualError(t, err, "no changes")
- })
-
- t.Run("no fields change", func(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &CollectorSetConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
-
- svc := Service{
- cfg: cfg,
- svc: &Services{},
- }
-
- current := collector.Collector{
- Fields: map[string]uuid.UUID{
- "example": uuid.New(),
- },
- }
- params := SetParams{
- Fields: &map[string]uuid.UUID{
- "example": current.Fields["example"],
- },
- }
- _, err = svc.getSetParams(ctx, current.ClientID, ¤t, ¶ms)
+ _, err = svc.getSetParams(current.ClientID, ¤t, ¶ms)
assert.EqualError(t, err, "no changes")
})
}
func TestSubmitSet(t *testing.T) {
t.Run("all fields", func(t *testing.T) {
-
ctx := t.Context()
pool, err := pgxmock.NewPool()
@@ -181,10 +129,6 @@ func TestSubmitSet(t *testing.T) {
LatestVersion: 1,
MinCleanVersion: 1,
MinTextVersion: 2,
- Fields: map[string]uuid.UUID{
- "original_key": uuid.New(),
- "og_key": uuid.New(),
- },
}
aV := int32(2)
params := dbSetParams{
@@ -192,12 +136,6 @@ func TestSubmitSet(t *testing.T) {
ActiveVersion: &aV,
MinCleanVersion: &minCleanV,
MinTextVersion: &minTextV,
- Fields: &map[string]uuid.UUID{
- "example_key": uuid.New(),
- "second_key": uuid.New(),
- "changed_key": current.Fields["original_key"],
- "og_key": current.Fields["og_key"],
- },
}
pool.ExpectBeginTx(pgx.TxOptions{})
@@ -212,22 +150,12 @@ func TestSubmitSet(t *testing.T) {
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(current.ClientID, rv, *params.MinTextVersion).
WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectExec("name: RemoveCollectorQuery :exec").WithArgs(&rv, current.Fields["original_key"], current.ClientID).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.MatchExpectationsInOrder(false)
- pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(current.ClientID, "example_key", (*params.Fields)["example_key"], int32(2)).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(current.ClientID, "second_key", (*params.Fields)["second_key"], int32(2)).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(current.ClientID, "changed_key", (*params.Fields)["changed_key"], int32(2)).
- WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
- err = svc.submitSet(ctx, ¤t, ¶ms)
+ err = svc.submitSet(ctx, ¶ms)
require.NoError(t, err)
})
t.Run("only active version", func(t *testing.T) {
-
ctx := t.Context()
pool, err := pgxmock.NewPool()
@@ -247,7 +175,6 @@ func TestSubmitSet(t *testing.T) {
LatestVersion: 1,
MinCleanVersion: 1,
MinTextVersion: 2,
- Fields: map[string]uuid.UUID{},
}
av := int32(2)
params := dbSetParams{
@@ -260,7 +187,7 @@ func TestSubmitSet(t *testing.T) {
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
- err = svc.submitSet(ctx, ¤t, ¶ms)
+ err = svc.submitSet(ctx, ¶ms)
require.NoError(t, err)
})
}
@@ -438,168 +365,6 @@ func TestNormalizeCodeVersions(t *testing.T) {
})
}
-func TestGetRemoveFields(t *testing.T) {
- current := map[string]uuid.UUID{}
- var update *map[string]uuid.UUID
-
- remove := getRemoveFields(current, update)
- assert.Len(t, remove, 0)
-
- current["a"] = uuid.New()
- remove = getRemoveFields(current, update)
- assert.Len(t, remove, 0)
-
- update = &map[string]uuid.UUID{
- "b": uuid.New(),
- }
-
- (*update)["a"] = current["a"]
- remove = getRemoveFields(current, update)
- assert.Len(t, remove, 0)
-
- (*update)["a"] = uuid.New()
- remove = getRemoveFields(current, update)
- assert.Len(t, remove, 1)
- assert.ElementsMatch(t, []uuid.UUID{current["a"]}, remove)
-
- (*update)["a"] = uuid.Nil
- (*update)["c"] = current["a"]
- remove = getRemoveFields(current, update)
- assert.Len(t, remove, 1)
- assert.ElementsMatch(t, []uuid.UUID{current["a"]}, remove)
-}
-
-func TestGetAddFields(t *testing.T) {
- current := map[string]uuid.UUID{}
- var update *map[string]uuid.UUID
-
- add := getAddFields(current, update)
- assert.Len(t, add, 0)
-
- current["a"] = uuid.New()
- add = getAddFields(current, update)
- assert.Len(t, add, 0)
-
- update = &map[string]uuid.UUID{}
-
- add = getAddFields(current, update)
- assert.Len(t, add, 0)
-
- (*update)["a"] = current["a"]
- add = getAddFields(current, update)
- assert.Len(t, add, 0)
-
- (*update)["a"] = uuid.New()
- add = getAddFields(current, update)
- assert.Len(t, add, 1)
- assert.Equal(t, map[string]uuid.UUID{
- "a": (*update)["a"],
- }, add)
-
- (*update)["a"] = current["a"]
- (*update)["b"] = uuid.New()
- add = getAddFields(current, update)
- assert.Len(t, add, 1)
- assert.Equal(t, map[string]uuid.UUID{
- "b": (*update)["b"],
- }, add)
-
- current = map[string]uuid.UUID{}
- add = getAddFields(current, update)
- assert.Len(t, add, 2)
- assert.Equal(t, map[string]uuid.UUID{
- "b": (*update)["b"],
- "a": (*update)["a"],
- }, add)
-}
-
-func TestNormalizeSetFieldsToDB(t *testing.T) {
- t.Run("valid key", func(t *testing.T) {
-
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &CollectorSetConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
-
- svc := Service{
- cfg: cfg,
- svc: &Services{
- Collector: collector.New(cfg),
- },
- }
-
- current := map[string]uuid.UUID{}
- fields := map[string]uuid.UUID{
- "example_key": uuid.New(),
- }
-
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs([]uuid.UUID{fields["example_key"]}).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(true),
- )
-
- dbparams, err := svc.normalizeSetFieldsToDB(ctx, current, &fields)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &map[string]uuid.UUID{
- "example_key": fields["example_key"],
- }, dbparams)
- })
-
- t.Run("duplicate values", func(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &CollectorSetConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
-
- svc := Service{
- cfg: cfg,
- svc: &Services{},
- }
-
- current := map[string]uuid.UUID{}
- fields := map[string]uuid.UUID{
- "example_key": uuid.New(),
- }
- fields["second_key"] = fields["example_key"]
-
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs([]uuid.UUID{fields["example_key"]}).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(true),
- )
-
- _, err = svc.normalizeSetFieldsToDB(ctx, current, &fields)
- assert.Error(t, err)
- })
- t.Run("no changes", func(t *testing.T) {
- ctx := t.Context()
- cfg := &CollectorSetConfig{}
-
- svc := Service{
- cfg: cfg,
- svc: &Services{
- Collector: collector.New(cfg),
- },
- }
-
- current := map[string]uuid.UUID{
- "example_key": uuid.New(),
- }
- fields := map[string]uuid.UUID{
- "example_key": current["example_key"],
- }
-
- val, err := svc.normalizeSetFieldsToDB(ctx, current, &fields)
- require.NoError(t, err)
- assert.Nil(t, val)
- })
-}
-
func TestInformSet(t *testing.T) {
t.Run("with version update", func(t *testing.T) {
ctx := t.Context()
@@ -656,43 +421,3 @@ func TestInformSet(t *testing.T) {
require.NoError(t, err)
})
}
-
-func TestNormalizeFieldsToDB(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &CollectorSetConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
-
- svc := Service{
- cfg: cfg,
- svc: &Services{},
- }
-
- fields := map[string]uuid.UUID{
- "example_key": uuid.New(),
- }
-
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs([]uuid.UUID{fields["example_key"]}).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(true),
- )
-
- dbparams, err := svc.normalizeFieldsToDB(ctx, &fields)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &map[string]uuid.UUID{
- "example_key": fields["example_key"],
- }, dbparams)
-
- fields["second_key"] = fields["example_key"]
-
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs([]uuid.UUID{fields["example_key"]}).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(true),
- )
-
- _, err = svc.normalizeFieldsToDB(ctx, &fields)
- assert.Error(t, err)
-}
diff --git a/internal/database/migrations/00000000000119_remove_queries.down.sql b/internal/database/migrations/00000000000119_remove_queries.down.sql
new file mode 100644
index 00000000..2ce31be9
--- /dev/null
+++ b/internal/database/migrations/00000000000119_remove_queries.down.sql
@@ -0,0 +1,287 @@
+-- Down migration to restore query functionality
+-- This recreates all query-related tables, views, and functions
+
+-- Drop the simplified fullActiveCollectors view
+DROP VIEW IF EXISTS fullActiveCollectors;
+
+-- Recreate query type enum
+CREATE TYPE queryType AS ENUM ('context_full', 'json_extractor');
+
+-- Recreate queries table
+CREATE TABLE queries (
+ queryId uuid primary key DEFAULT uuid_generate_v7(),
+ queryType queryType not null
+);
+
+-- Recreate queryVersions table
+CREATE TABLE queryVersions (
+ queryId uuid not null,
+ versionId int not null,
+ addedAt timestamp not null default current_timestamp,
+ primary key (versionId, queryId),
+ foreign key (queryId) references queries(queryId)
+);
+
+-- Recreate version number function and trigger
+CREATE OR REPLACE FUNCTION setQueryVersionNumber()
+RETURNS TRIGGER AS $$
+BEGIN
+ SELECT COALESCE(MAX(versionId), 0) + 1
+ INTO NEW.versionId
+ FROM queryVersions
+ WHERE queryId = NEW.queryId;
+
+ RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER setQueryVersionNumberTrigger
+BEFORE INSERT ON queryVersions
+FOR EACH ROW
+EXECUTE FUNCTION setQueryVersionNumber();
+
+-- Recreate queryActiveVersions table
+CREATE TABLE queryActiveVersions (
+ activeVersionEntryId uuid primary key DEFAULT uuid_generate_v7(),
+ queryId uuid not null,
+ versionId int not null,
+ foreign key (queryId, versionId) references queryVersions(queryId, versionId)
+);
+
+-- Recreate requiredQueries table
+CREATE TABLE requiredQueries (
+ requiredQueryEntryId uuid primary key DEFAULT uuid_generate_v7(),
+ queryId uuid not null,
+ requiredQueryId uuid not null,
+ addedVersion int not null,
+ removedVersion int,
+ foreign key (queryId) references queries(queryId),
+ foreign key (requiredQueryId) references queries(queryId),
+ foreign key (queryId, addedVersion) references queryVersions(queryId, versionId),
+ foreign key (queryId, removedVersion) references queryVersions(queryId, versionId),
+ unique (queryId, requiredQueryId, removedVersion)
+);
+
+-- Recreate queryConfigs table
+CREATE TABLE queryConfigs (
+ configId uuid primary key DEFAULT uuid_generate_v7(),
+ queryId uuid not null,
+ config jsonb not null,
+ addedVersion int not null,
+ removedVersion int,
+ foreign key (queryId) references queries(queryId),
+ foreign key (queryId, addedVersion) references queryVersions(queryId, versionId),
+ foreign key (queryId, removedVersion) references queryVersions(queryId, versionId),
+ unique (queryId, removedVersion)
+);
+
+-- Recreate removeQueryConfig function and trigger
+CREATE OR REPLACE FUNCTION removeQueryConfig()
+RETURNS TRIGGER AS $$
+BEGIN
+ UPDATE queryConfigs
+ SET removedVersion = NEW.addedVersion
+ WHERE queryId = NEW.queryId and removedVersion is null;
+
+ RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER removeQueryConfigTrigger
+BEFORE INSERT ON queryConfigs
+FOR EACH ROW
+EXECUTE FUNCTION removeQueryConfig();
+
+-- Recreate collectorQueries table
+CREATE TABLE collectorQueries (
+ id uuid primary key DEFAULT uuid_generate_v7(),
+ clientId varchar(255) not null,
+ name varchar(255) not null,
+ queryId uuid not null,
+ addedVersion int not null,
+ removedVersion int,
+ foreign key (queryId) references queries(queryId),
+ foreign key (clientId) references clients(clientId),
+ foreign key (clientId, addedVersion) references collectorVersions(clientId, id),
+ foreign key (clientId, removedVersion) references collectorVersions(clientId, id),
+ unique (clientId, name, removedVersion)
+);
+
+-- Recreate results table
+CREATE TABLE results (
+ id uuid primary key DEFAULT uuid_generate_v7(),
+ textEntryId uuid not null,
+ queryId uuid not null,
+ value TEXT not null,
+ queryVersion int not null,
+ foreign key (queryId) references queries(queryId),
+ foreign key (textEntryId) references documentTextExtractions(id),
+ foreign key (queryId, queryVersion) references queryVersions(queryId, versionId)
+);
+
+-- Recreate resultDependencies table
+CREATE TABLE resultDependencies (
+ resultId uuid not null,
+ requiredResultId uuid not null,
+ foreign key (resultId) references results(id),
+ foreign key (requiredResultId) references results(id),
+ CONSTRAINT result_not_self_dependent CHECK (resultId != requiredResultId)
+);
+
+-- Recreate query views
+CREATE VIEW queryCurrentActiveVersions as
+SELECT DISTINCT
+ q.queryId,
+ coalesce(
+ (FIRST_VALUE(av.versionId) OVER (PARTITION BY q.queryId ORDER BY av.activeVersionEntryId DESC)),
+ 0
+ )::int as activeVersion
+ FROM queries AS q
+ LEFT JOIN queryActiveVersions as av on av.queryId = q.queryId;
+
+CREATE VIEW queryLatestVersions as
+ SELECT
+ q.queryId,
+ coalesce(max(v.versionId), 0)::int as latestVersion
+ FROM queries AS q
+ LEFT JOIN queryVersions as v on v.queryId = q.queryId
+ GROUP BY q.queryId;
+
+CREATE VIEW queryCurrentConfigs as
+ SELECT av.queryId, c.config
+ FROM queryCurrentActiveVersions as av
+ LEFT JOIN queryConfigs AS c ON av.queryId = c.queryId
+ and isInVersion(av.activeVersion, c.addedVersion, c.removedVersion);
+
+CREATE VIEW queryCurrentRequiredIds as
+ SELECT DISTINCT av.queryId, r.requiredQueryId
+ FROM queryCurrentActiveVersions as av
+ LEFT JOIN requiredQueries AS r ON av.queryId = r.queryId
+ and isInVersion(av.activeVersion, r.addedVersion, r.removedVersion);
+
+CREATE VIEW queryCurrentRequiredIdsAGG as
+ SELECT queryId,
+ coalesce(
+ ARRAY_AGG(DISTINCT requiredQueryId)
+ FILTER (WHERE requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
+ array[]::uuid[]
+ )::uuid[] as requiredIds
+ FROM queryCurrentRequiredIds
+ GROUP BY queryId;
+
+CREATE VIEW fullActiveQueries AS
+SELECT DISTINCT q.queryId, q.queryType, av.activeVersion, lv.latestVersion, c.config, r.requiredIds
+ FROM queries AS q
+ JOIN queryCurrentActiveVersions as av on q.queryId = av.queryId
+ JOIN queryLatestVersions as lv on lv.queryId = q.queryId
+ JOIN queryCurrentConfigs AS c ON q.queryId = c.queryId
+ JOIN queryCurrentRequiredIdsAGG AS r ON q.queryId = r.queryId;
+
+CREATE VIEW queryActiveDependencies AS
+WITH RECURSIVE queryActiveDependencies(queryId, requiredQueryId, path, cycle) AS (
+ SELECT
+ queryId,
+ requiredQueryId,
+ ARRAY[queryId, requiredQueryId]::uuid[] AS path,
+ false AS cycle
+ FROM queryCurrentRequiredIds
+
+ UNION ALL
+
+ SELECT
+ q.queryId,
+ qd.requiredQueryId,
+ path || qd.requiredQueryId,
+ qd.requiredQueryId = ANY(path) AS cycle
+ FROM queryCurrentRequiredIds as q
+ JOIN queryActiveDependencies as qd ON q.queryId = qd.requiredQueryId
+ WHERE NOT qd.cycle
+ )
+ SELECT DISTINCT queryId as id, requiredQueryId
+ FROM queryActiveDependencies
+ WHERE NOT cycle;
+
+-- Recreate collector views that depend on queries
+CREATE VIEW currentCollectorQueries as
+SELECT DISTINCT
+ av.clientId,
+ q.name,
+ q.queryId
+ FROM collectorCurrentActiveVersions as av
+ LEFT JOIN collectorQueries AS q ON av.clientId = q.clientId
+ and isInVersion(av.activeVersion, q.addedVersion, q.removedVersion);
+
+CREATE VIEW currentCollectorQueriesJSONAGG as
+SELECT DISTINCT
+ clientId,
+ jsonb_object_agg(name, queryId) FILTER (WHERE name is not null) AS fields
+ FROM currentCollectorQueries
+ GROUP BY clientId;
+
+CREATE VIEW fullActiveCollectors AS
+ SELECT DISTINCT av.clientId,
+ ccv.minCleanVersion, ctv.minTextVersion,
+ av.activeVersion, lv.latestVersion,
+ q.fields
+ FROM collectorCurrentActiveVersions as av
+ JOIN collectorLatestVersions as lv on lv.clientId = av.clientId
+ JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId
+ JOIN currentCollectorMinTextVersions AS ctv ON av.clientId = ctv.clientId
+ JOIN currentCollectorQueriesJSONAGG AS q ON av.clientId = q.clientId;
+
+CREATE VIEW collectorQueryDependencyTree AS
+ WITH RECURSIVE collectorQueryDependencyTree AS (
+ SELECT cq.clientId, cq.queryId, ri.requiredIds
+ FROM currentCollectorQueries as cq
+ JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId
+
+ UNION ALL
+
+ SELECT acq.clientId, q.queryId, q.requiredIds
+ FROM queryCurrentRequiredIdsAGG as q
+ JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds)
+ )
+ SELECT DISTINCT ct.clientId, ct.queryId, q.queryType, av.activeVersion as queryVersion, ct.requiredIds
+ FROM collectorQueryDependencyTree as ct
+ JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId
+ JOIN queries as q on q.queryId = ct.queryId;
+
+CREATE OR REPLACE FUNCTION collectorQueryDependencyTreeByClient(
+ _clientId varchar(255)
+)
+RETURNS TABLE (
+ clientId varchar(255),
+ queryId uuid,
+ queryType queryType,
+ queryVersion int,
+ requiredIds uuid[]
+) AS $$
+BEGIN
+ RETURN QUERY
+ WITH clientQueries as (
+ SELECT q.clientId, q.queryId
+ FROM currentCollectorQueries as q
+ WHERE q.clientId = _clientId
+ ),
+ dependencyTree as (
+ WITH RECURSIVE collectorQueryDependencyTree AS (
+ SELECT cq.clientId, cq.queryId, ri.requiredIds
+ FROM clientQueries as cq
+ JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId
+
+ UNION ALL
+
+ SELECT acq.clientId, q.queryId, q.requiredIds
+ FROM queryCurrentRequiredIdsAGG as q
+ JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds)
+ )
+ SELECT DISTINCT ct.clientId, ct.queryId, q.queryType, av.activeVersion as queryVersion, ct.requiredIds
+ FROM collectorQueryDependencyTree as ct
+ JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId
+ JOIN queries as q on q.queryId = ct.queryId
+ )
+ SELECT t.clientId, t.queryId, t.queryType, t.queryVersion, t.requiredIds
+ FROM dependencyTree as t
+ WHERE t.clientID is not null;
+END;
+$$ LANGUAGE plpgsql;
diff --git a/internal/database/migrations/00000000000119_remove_queries.up.sql b/internal/database/migrations/00000000000119_remove_queries.up.sql
new file mode 100644
index 00000000..7a4eb28d
--- /dev/null
+++ b/internal/database/migrations/00000000000119_remove_queries.up.sql
@@ -0,0 +1,53 @@
+-- Migration to remove query functionality from the database
+-- This drops all query-related tables, views, and functions
+
+-- First drop the function that depends on query views
+DROP FUNCTION IF EXISTS collectorQueryDependencyTreeByClient;
+
+-- Drop the collector views that depend on query tables (in order of dependencies)
+DROP VIEW IF EXISTS collectorQueryDependencyTree;
+DROP VIEW IF EXISTS fullActiveCollectors;
+DROP VIEW IF EXISTS currentCollectorQueriesJSONAGG;
+DROP VIEW IF EXISTS currentCollectorQueries;
+
+-- Drop query views (in order of dependencies)
+DROP VIEW IF EXISTS fullActiveQueries;
+DROP VIEW IF EXISTS queryActiveDependencies;
+DROP VIEW IF EXISTS queryCurrentRequiredIdsAGG;
+DROP VIEW IF EXISTS queryCurrentRequiredIds;
+DROP VIEW IF EXISTS queryCurrentConfigs;
+DROP VIEW IF EXISTS queryLatestVersions;
+DROP VIEW IF EXISTS queryCurrentActiveVersions;
+
+-- Drop result tables
+DROP TABLE IF EXISTS resultDependencies;
+DROP TABLE IF EXISTS results;
+
+-- Drop collectorQueries table (links collectors to queries)
+DROP TABLE IF EXISTS collectorQueries;
+
+-- Drop query configuration and dependency tables
+DROP TABLE IF EXISTS queryConfigs;
+DROP TABLE IF EXISTS requiredQueries;
+DROP TABLE IF EXISTS queryActiveVersions;
+DROP TABLE IF EXISTS queryVersions;
+DROP TABLE IF EXISTS queries;
+
+-- Drop triggers and functions
+DROP TRIGGER IF EXISTS setQueryVersionNumberTrigger ON queryVersions;
+DROP FUNCTION IF EXISTS setQueryVersionNumber;
+DROP TRIGGER IF EXISTS removeQueryConfigTrigger ON queryConfigs;
+DROP FUNCTION IF EXISTS removeQueryConfig;
+
+-- Drop the query type enum
+DROP TYPE IF EXISTS queryType;
+
+-- Recreate the fullActiveCollectors view without the fields column
+CREATE VIEW fullActiveCollectors AS
+ SELECT DISTINCT av.clientId,
+ ccv.minCleanVersion, ctv.minTextVersion,
+ av.activeVersion, lv.latestVersion
+ FROM collectorCurrentActiveVersions as av
+ JOIN collectorLatestVersions as lv on lv.clientId = av.clientId
+ JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId
+ JOIN currentCollectorMinTextVersions AS ctv ON av.clientId = ctv.clientId;
diff --git a/internal/database/queries/client.sql b/internal/database/queries/client.sql
index 08f7234a..75f0c39b 100644
--- a/internal/database/queries/client.sql
+++ b/internal/database/queries/client.sql
@@ -15,6 +15,9 @@ UPDATE clients SET name = $1 WHERE clientId = $2;
INSERT INTO clientCanSync (canSync, clientId) VALUES ($1, $2);
-- name: IsClientSynced :one
+-- Query functionality has been removed. A client is considered synced when
+-- all documents have completed text extraction (or failed with explicit failure).
+-- See remove_query_plan.md for details.
WITH
docs AS (
-- Get all documents for this client
@@ -42,46 +45,6 @@ doc_text_entries AS (
FROM
doc_clean_entries d
LEFT JOIN currentTextEntries cte ON cte.cleanId = d.clean_entry_id
-),
-required_results AS (
- -- All required document-query-version combinations
- SELECT
- dte.document_id,
- cqdt.queryId,
- cqdt.queryVersion,
- dte.text_entry_id
- FROM
- doc_text_entries dte
- JOIN collectorQueryDependencyTreeByClient($1) cqdt ON cqdt.clientId = dte.client_id
- where dte.clean_fail is null and dte.text_entry_id IS NOT NULL
-),
-existing_results AS (
- -- Valid results that exist
- SELECT
- rr.document_id AS document_id,
- r.queryId,
- r.id AS result_id
- FROM
- results r
- JOIN required_results rr ON
- r.queryId = rr.queryId AND
- r.queryVersion = rr.queryVersion AND
- r.textEntryId = rr.text_entry_id
-),
-missing_results AS (
- -- Find missing results
- SELECT rr.queryId
- FROM required_results rr
- LEFT JOIN existing_results er on er.queryId = rr.queryId
- WHERE er.result_id is null
-),
-dependency_check AS (
- -- Check for missing dependencies
- SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId
- FROM existing_results er
- JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId
- LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id
- WHERE rd.resultId is null
)
SELECT (
-- No documents means client is synced
@@ -89,22 +52,11 @@ SELECT (
OR
- -- Documents with no text entries
- (
- NOT EXISTS (
- SELECT 1 FROM doc_text_entries
- where clean_entry_id is null or
- (clean_fail is null and text_entry_id is null)
- )
-
- and
-
- -- Documents with missing results
- NOT EXISTS (SELECT 1 FROM missing_results)
-
- and
-
- -- Documents with missing dependencies
- NOT EXISTS (SELECT 1 FROM dependency_check)
- )::bool
+ -- All documents have completed processing (clean entry exists and either
+ -- text entry exists or clean failed)
+ NOT EXISTS (
+ SELECT 1 FROM doc_text_entries
+ where clean_entry_id is null or
+ (clean_fail is null and text_entry_id is null)
+ )
)::bool as is_synced;
diff --git a/internal/database/queries/collector.sql b/internal/database/queries/collector.sql
index 226cad1e..eeae2028 100644
--- a/internal/database/queries/collector.sql
+++ b/internal/database/queries/collector.sql
@@ -1,7 +1,3 @@
--- name: ListCollectorQueries :many
-SELECT clientId, queryId, queryType, queryVersion, requiredIds
- FROM collectorQueryDependencyTreeByClient(@clientId);
-
-- name: GetCollectorByClientID :one
SELECT * FROM fullActiveCollectors WHERE clientId = @clientId LIMIT 1;
@@ -16,9 +12,3 @@ INSERT INTO collectorMinCleanVersions (clientId, addedVersion, versionId) VALUES
-- name: SetCollectorTextVersion :exec
INSERT INTO collectorMinTextVersions (clientId, addedVersion, versionId) VALUES ($1, $2, $3);
-
--- name: AddCollectorQuery :exec
-INSERT INTO collectorQueries (clientId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4);
-
--- name: RemoveCollectorQuery :exec
-UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and clientId = $3 and removedVersion is null;
diff --git a/internal/database/queries/document.sql b/internal/database/queries/document.sql
index 10b8f5f2..7b036b01 100644
--- a/internal/database/queries/document.sql
+++ b/internal/database/queries/document.sql
@@ -2,30 +2,13 @@
SELECT id, clientId, hash FROM documents WHERE id = $1;
-- name: GetDocumentExternal :one
-WITH
-docs AS (
- SELECT id, hash, clientId
- FROM documents
- WHERE id = @documentId
-),
-namedResults AS (
- SELECT
- q.name,
- dd.id,
- d.value
- FROM currentCollectorQueries AS q
- JOIN docs as dd on dd.clientId = q.clientId
- LEFT JOIN listValidDocumentResults(@documentId) AS d
- ON d.queryId = q.queryId and q.name is not null
-)
+-- Query functionality has been removed. See remove_query_plan.md for details.
SELECT
- d.id,
- d.clientId,
- d.hash,
- COALESCE(jsonb_object_agg(r.name, r.value) FILTER (WHERE r.name IS NOT NULL), '{}') AS fields
-FROM docs AS d
-LEFT JOIN namedResults AS r ON d.id = r.id
-GROUP BY d.id, d.clientId, d.hash;
+ id,
+ clientId,
+ hash
+FROM documents
+WHERE id = @documentId;
-- name: ListDocumentsByClient :many
SELECT id, hash from documents where clientId = @clientId;
diff --git a/internal/database/queries/query.sql b/internal/database/queries/query.sql
deleted file mode 100644
index cc2bc306..00000000
--- a/internal/database/queries/query.sql
+++ /dev/null
@@ -1,86 +0,0 @@
--- name: GetActiveQueryConfig :one
-SELECT config FROM queryCurrentConfigs where queryId = $1;
-
--- name: GetQuery :one
-SELECT * FROM fullActiveQueries WHERE queryId = $1;
-
--- name: GetQueryWithVersion :one
-WITH query as (
- SELECT queryId, queryType FROM queries WHERE queryId = @id
-),
-config as (
-SELECT c.queryId, c.config
- FROM query AS q
- LEFT JOIN queryConfigs AS c ON q.queryId = c.queryId
- and isInVersion(@version, c.addedVersion, c.removedVersion)
-),
-requiredIds as (
- SELECT r.queryId,
- coalesce(
- ARRAY_AGG(DISTINCT r.requiredQueryId)
- FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
- array[]::uuid[]
- )::uuid[] as requiredIds
- FROM query AS q
- LEFT JOIN requiredQueries AS r ON q.queryId = r.queryId
- and isInVersion(@version, r.addedVersion, r.removedVersion)
- GROUP BY r.queryId
-)
-SELECT DISTINCT q.queryId, q.queryType,
- av.activeVersion,
- lv.latestVersion, c.config,
- r.requiredIds
- FROM query AS q
- JOIN queryCurrentActiveVersions as av on q.queryId = av.queryId
- JOIN queryLatestVersions as lv on lv.queryId = q.queryId
- LEFT JOIN config AS c ON q.queryId = c.queryId
- LEFT JOIN requiredIds AS r ON q.queryId = r.queryId;
-
--- name: ListQueries :many
-SELECT * FROM fullActiveQueries;
-
--- name: ListQueriesById :many
-SELECT * FROM fullActiveQueries WHERE queryId = any($1);
-
--- name: CreateQuery :one
-INSERT INTO queries (queryType) VALUES ($1) RETURNING queryId;
-
--- name: AddLatestQueryVersion :one
-INSERT INTO queryVersions (queryId) VALUES ($1) RETURNING versionId;
-
--- name: AddActiveQueryVersion :exec
-INSERT INTO queryActiveVersions (queryId, versionId) VALUES ($1, $2);
-
--- name: AddRequiredQuery :exec
-INSERT INTO requiredQueries (queryId, requiredQueryId, addedVersion) VALUES ($1, $2, $3);
-
--- name: RemoveRequiredQuery :exec
-UPDATE requiredQueries SET removedVersion = $1 WHERE requiredQueryId = $2 and queryId = $3 and removedVersion is null;
-
--- name: SetQueryConfig :exec
-INSERT INTO queryConfigs (queryId, config, addedVersion) VALUES ($1, $2, $3);
-
--- name: AllQueriesExist :one
-SELECT COUNT(*) = COUNT(DISTINCT queryId) AS all_exist
- FROM unnest($1::uuid[]) AS input_id
- LEFT JOIN queries ON input_id = queries.queryId;
-
--- name: IsQueryInDependencyTree :one
-SELECT EXISTS (
- SELECT 1 FROM queryActiveDependencies
- WHERE id = any(@requiredQueryIds)
- and requiredQueryId = @queryId
- or @queryId = any(@requiredQueryIds)
-);
-
--- name: ListQueryDirectDependentsByDocumentID :many
-WITH doc AS (
- SELECT clientId FROM documents WHERE id = @documentId
-)
-SELECT dt.queryId
-FROM doc,
- LATERAL collectorQueryDependencyTreeByClient(doc.clientId) AS dt
-WHERE @queryId = ANY(dt.requiredIds);
-
--- name: ListQueryClientIDs :many
-SELECT clientId FROM collectorQueryDependencyTree WHERE queryId = $1;
diff --git a/internal/database/queries/result.sql b/internal/database/queries/result.sql
deleted file mode 100644
index a0313b4f..00000000
--- a/internal/database/queries/result.sql
+++ /dev/null
@@ -1,92 +0,0 @@
--- name: ListQueryRequirementValues :many
-WITH reqQueries as (
- SELECT av.queryId, av.activeVersion, q.queryType
- FROM requiredQueries as rq
- JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId
- JOIN queries as q on q.queryId = av.queryId
- WHERE rq.queryId = @queryId
- and isInVersion(@version, rq.addedVersion, rq.removedVersion)
-),
-docs as (
- SELECT id, clientId
- FROM documents
- WHERE id = @documentId
-),
-codeVersions as (
- SELECT
- d.id as documentId,
- mcv.minCleanVersion,
- mtv.minTextVersion
- FROM docs as d
- JOIN currentCollectorMinTextVersions as mtv on mtv.clientId = d.clientId
- JOIN currentCollectorMinCleanVersions as mcv on mcv.clientId = d.clientId
-),
-latestVersions AS (
- SELECT
- r.id,
- rq.queryId,
- rq.queryType,
- r.queryVersion,
- r.textEntryId,
- r.value,
- ROW_NUMBER() OVER (
- PARTITION BY r.queryId
- ORDER BY r.id DESC
- ) as rowNumber
- FROM reqQueries as rq
- JOIN currentTextEntries as cte on cte.documentId = @documentId
- LEFT JOIN results as r ON rq.queryId = r.queryId
- and r.queryVersion = rq.activeVersion
- and cte.id = r.textEntryId
-)
-SELECT DISTINCT id, queryId, queryType, value
-FROM latestVersions
-WHERE rowNumber = 1;
-
--- name: AddResult :one
-INSERT INTO results (queryId, value, textEntryId, queryVersion) VALUES ($1, $2, $3, $4) returning id;
-
--- name: AddResultDependency :exec
-INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2);
-
--- name: GetResultValueWithVersion :one
-WITH doc as (
- SELECT id, clientId
- FROM documents
- WHERE id = @documentId
-)
-SELECT r.id, r.value
- FROM doc as d
- JOIN currentTextEntries as cte on cte.documentId = d.id
- LEFT JOIN results as r
- on r.queryId = @queryId
- and r.queryVersion = @queryVersion
- and r.textEntryId = cte.id;
-
--- name: ListUnsyncedNoDepsQueriesByDocId :many
-WITH docs as (
- SELECT id, clientId from documents where id = $1
-),
-dependency_tree as (
- select q.queryId, q.requiredIds, q.clientId, q.queryVersion
- from docs,
- LATERAL collectorQueryDependencyTreeByClient(docs.clientId) AS q
-),
-text_entries as (
- select id, documentId from currentTextEntries where documentId = $1
-),
-unsyncedQueries AS (
- SELECT DISTINCT dt.queryId, dt.requiredIds
- from docs as d
- JOIN dependency_tree as dt on d.clientId = dt.clientId
- JOIN text_entries as cte on cte.documentId = d.id
- LEFT JOIN results as r
- on r.queryId = dt.queryId
- and r.queryVersion = dt.queryVersion
- and cte.id = r.textEntryId
- where r.value is null
-)
-SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
- WHERE NOT EXISTS (
- SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds)
- );
diff --git a/internal/database/repository/client.sql.go b/internal/database/repository/client.sql.go
index 1d0677fe..bd297bee 100644
--- a/internal/database/repository/client.sql.go
+++ b/internal/database/repository/client.sql.go
@@ -85,46 +85,6 @@ doc_text_entries AS (
FROM
doc_clean_entries d
LEFT JOIN currentTextEntries cte ON cte.cleanId = d.clean_entry_id
-),
-required_results AS (
- -- All required document-query-version combinations
- SELECT
- dte.document_id,
- cqdt.queryId,
- cqdt.queryVersion,
- dte.text_entry_id
- FROM
- doc_text_entries dte
- JOIN collectorQueryDependencyTreeByClient($1) cqdt ON cqdt.clientId = dte.client_id
- where dte.clean_fail is null and dte.text_entry_id IS NOT NULL
-),
-existing_results AS (
- -- Valid results that exist
- SELECT
- rr.document_id AS document_id,
- r.queryId,
- r.id AS result_id
- FROM
- results r
- JOIN required_results rr ON
- r.queryId = rr.queryId AND
- r.queryVersion = rr.queryVersion AND
- r.textEntryId = rr.text_entry_id
-),
-missing_results AS (
- -- Find missing results
- SELECT rr.queryId
- FROM required_results rr
- LEFT JOIN existing_results er on er.queryId = rr.queryId
- WHERE er.result_id is null
-),
-dependency_check AS (
- -- Check for missing dependencies
- SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId
- FROM existing_results er
- JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId
- LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id
- WHERE rd.resultId is null
)
SELECT (
-- No documents means client is synced
@@ -132,28 +92,19 @@ SELECT (
OR
- -- Documents with no text entries
- (
- NOT EXISTS (
- SELECT 1 FROM doc_text_entries
- where clean_entry_id is null or
- (clean_fail is null and text_entry_id is null)
- )
-
- and
-
- -- Documents with missing results
- NOT EXISTS (SELECT 1 FROM missing_results)
-
- and
-
- -- Documents with missing dependencies
- NOT EXISTS (SELECT 1 FROM dependency_check)
- )::bool
+ -- All documents have completed processing (clean entry exists and either
+ -- text entry exists or clean failed)
+ NOT EXISTS (
+ SELECT 1 FROM doc_text_entries
+ where clean_entry_id is null or
+ (clean_fail is null and text_entry_id is null)
+ )
)::bool as is_synced
`
-// IsClientSynced
+// Query functionality has been removed. A client is considered synced when
+// all documents have completed text extraction (or failed with explicit failure).
+// See remove_query_plan.md for details.
//
// WITH
// docs AS (
@@ -182,46 +133,6 @@ SELECT (
// FROM
// doc_clean_entries d
// LEFT JOIN currentTextEntries cte ON cte.cleanId = d.clean_entry_id
-// ),
-// required_results AS (
-// -- All required document-query-version combinations
-// SELECT
-// dte.document_id,
-// cqdt.queryId,
-// cqdt.queryVersion,
-// dte.text_entry_id
-// FROM
-// doc_text_entries dte
-// JOIN collectorQueryDependencyTreeByClient($1) cqdt ON cqdt.clientId = dte.client_id
-// where dte.clean_fail is null and dte.text_entry_id IS NOT NULL
-// ),
-// existing_results AS (
-// -- Valid results that exist
-// SELECT
-// rr.document_id AS document_id,
-// r.queryId,
-// r.id AS result_id
-// FROM
-// results r
-// JOIN required_results rr ON
-// r.queryId = rr.queryId AND
-// r.queryVersion = rr.queryVersion AND
-// r.textEntryId = rr.text_entry_id
-// ),
-// missing_results AS (
-// -- Find missing results
-// SELECT rr.queryId
-// FROM required_results rr
-// LEFT JOIN existing_results er on er.queryId = rr.queryId
-// WHERE er.result_id is null
-// ),
-// dependency_check AS (
-// -- Check for missing dependencies
-// SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId
-// FROM existing_results er
-// JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId
-// LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id
-// WHERE rd.resultId is null
// )
// SELECT (
// -- No documents means client is synced
@@ -229,24 +140,13 @@ SELECT (
//
// OR
//
-// -- Documents with no text entries
-// (
-// NOT EXISTS (
-// SELECT 1 FROM doc_text_entries
-// where clean_entry_id is null or
-// (clean_fail is null and text_entry_id is null)
-// )
-//
-// and
-//
-// -- Documents with missing results
-// NOT EXISTS (SELECT 1 FROM missing_results)
-//
-// and
-//
-// -- Documents with missing dependencies
-// NOT EXISTS (SELECT 1 FROM dependency_check)
-// )::bool
+// -- All documents have completed processing (clean entry exists and either
+// -- text entry exists or clean failed)
+// NOT EXISTS (
+// SELECT 1 FROM doc_text_entries
+// where clean_entry_id is null or
+// (clean_fail is null and text_entry_id is null)
+// )
// )::bool as is_synced
func (q *Queries) IsClientSynced(ctx context.Context, dollar_1 *string) (bool, error) {
row := q.db.QueryRow(ctx, isClientSynced, dollar_1)
diff --git a/internal/database/repository/collector.sql.go b/internal/database/repository/collector.sql.go
index 5fea2f78..0a70cdc3 100644
--- a/internal/database/repository/collector.sql.go
+++ b/internal/database/repository/collector.sql.go
@@ -7,34 +7,8 @@ package repository
import (
"context"
-
- "github.com/google/uuid"
)
-const addCollectorQuery = `-- name: AddCollectorQuery :exec
-INSERT INTO collectorQueries (clientId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
-`
-
-type AddCollectorQueryParams struct {
- Clientid string `db:"clientid"`
- Name string `db:"name"`
- Queryid uuid.UUID `db:"queryid"`
- Addedversion int32 `db:"addedversion"`
-}
-
-// AddCollectorQuery
-//
-// INSERT INTO collectorQueries (clientId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
-func (q *Queries) AddCollectorQuery(ctx context.Context, arg *AddCollectorQueryParams) error {
- _, err := q.db.Exec(ctx, addCollectorQuery,
- arg.Clientid,
- arg.Name,
- arg.Queryid,
- arg.Addedversion,
- )
- return err
-}
-
const addLatestCollectorVersion = `-- name: AddLatestCollectorVersion :one
INSERT INTO collectorVersions (clientId) VALUES ($1) RETURNING id
`
@@ -50,12 +24,12 @@ func (q *Queries) AddLatestCollectorVersion(ctx context.Context, clientid string
}
const getCollectorByClientID = `-- name: GetCollectorByClientID :one
-SELECT clientid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE clientId = $1 LIMIT 1
+SELECT clientid, mincleanversion, mintextversion, activeversion, latestversion FROM fullActiveCollectors WHERE clientId = $1 LIMIT 1
`
// GetCollectorByClientID
//
-// SELECT clientid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE clientId = $1 LIMIT 1
+// SELECT clientid, mincleanversion, mintextversion, activeversion, latestversion FROM fullActiveCollectors WHERE clientId = $1 LIMIT 1
func (q *Queries) GetCollectorByClientID(ctx context.Context, clientid string) (*Fullactivecollector, error) {
row := q.db.QueryRow(ctx, getCollectorByClientID, clientid)
var i Fullactivecollector
@@ -65,72 +39,10 @@ func (q *Queries) GetCollectorByClientID(ctx context.Context, clientid string) (
&i.Mintextversion,
&i.Activeversion,
&i.Latestversion,
- &i.Fields,
)
return &i, err
}
-const listCollectorQueries = `-- name: ListCollectorQueries :many
-SELECT clientId, queryId, queryType, queryVersion, requiredIds
- FROM collectorQueryDependencyTreeByClient($1)
-`
-
-type ListCollectorQueriesRow struct {
- Clientid *string `db:"clientid"`
- Queryid *uuid.UUID `db:"queryid"`
- Querytype NullQuerytype `db:"querytype"`
- Queryversion *int32 `db:"queryversion"`
- Requiredids []uuid.UUID `db:"requiredids"`
-}
-
-// ListCollectorQueries
-//
-// SELECT clientId, queryId, queryType, queryVersion, requiredIds
-// FROM collectorQueryDependencyTreeByClient($1)
-func (q *Queries) ListCollectorQueries(ctx context.Context, clientid string) ([]*ListCollectorQueriesRow, error) {
- rows, err := q.db.Query(ctx, listCollectorQueries, clientid)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
- items := []*ListCollectorQueriesRow{}
- for rows.Next() {
- var i ListCollectorQueriesRow
- if err := rows.Scan(
- &i.Clientid,
- &i.Queryid,
- &i.Querytype,
- &i.Queryversion,
- &i.Requiredids,
- ); err != nil {
- return nil, err
- }
- items = append(items, &i)
- }
- if err := rows.Err(); err != nil {
- return nil, err
- }
- return items, nil
-}
-
-const removeCollectorQuery = `-- name: RemoveCollectorQuery :exec
-UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and clientId = $3 and removedVersion is null
-`
-
-type RemoveCollectorQueryParams struct {
- Removedversion *int32 `db:"removedversion"`
- Queryid uuid.UUID `db:"queryid"`
- Clientid string `db:"clientid"`
-}
-
-// RemoveCollectorQuery
-//
-// UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and clientId = $3 and removedVersion is null
-func (q *Queries) RemoveCollectorQuery(ctx context.Context, arg *RemoveCollectorQueryParams) error {
- _, err := q.db.Exec(ctx, removeCollectorQuery, arg.Removedversion, arg.Queryid, arg.Clientid)
- return err
-}
-
const setActiveCollectorVersion = `-- name: SetActiveCollectorVersion :exec
INSERT INTO collectorActiveVersions (clientId, versionId) VALUES ($1, $2)
`
diff --git a/internal/database/repository/collector_test.go b/internal/database/repository/collector_test.go
deleted file mode 100644
index 20823bf0..00000000
--- a/internal/database/repository/collector_test.go
+++ /dev/null
@@ -1,263 +0,0 @@
-package repository_test
-
-import (
- "fmt"
- "testing"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/test"
-
- "github.com/google/uuid"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestCollector(t *testing.T) {
- t.Parallel()
- if testing.Short() {
- t.SkipNow()
- }
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- contextId, err := queries.CreateQuery(ctx, repository.QuerytypeContextFull)
- require.NoError(t, err)
- jsonId, err := queries.CreateQuery(ctx, repository.QuerytypeJsonExtractor)
- require.NoError(t, err)
- version, err := queries.AddLatestQueryVersion(ctx, jsonId)
- require.NoError(t, err)
- err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
- Queryid: jsonId,
- Versionid: version,
- })
- require.NoError(t, err)
- assert.Equal(t, int32(1), version)
- err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
- Queryid: jsonId,
- Requiredqueryid: contextId,
- Addedversion: version,
- })
- require.NoError(t, err)
-
- clientId := "EXAMPLE"
- err = queries.CreateClient(ctx, &repository.CreateClientParams{
- Name: "example_client",
- Clientid: clientId,
- })
- require.NoError(t, err)
- minCleanVersion := int64(2)
- minTextVersion := int64(4)
-
- coll, err := queries.GetCollectorByClientID(ctx, clientId)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivecollector{
- Clientid: clientId,
- Mincleanversion: 0,
- Mintextversion: 0,
- Activeversion: 0,
- Latestversion: 0,
- }, coll)
-
- version, err = queries.AddLatestCollectorVersion(ctx, clientId)
- require.NoError(t, err)
- assert.Equal(t, int32(1), version)
-
- coll, err = queries.GetCollectorByClientID(ctx, clientId)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivecollector{
- Clientid: clientId,
- Mincleanversion: 0,
- Mintextversion: 0,
- Activeversion: 0,
- Latestversion: 1,
- }, coll)
-
- err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
- Versionid: 1,
- Clientid: clientId,
- })
- require.NoError(t, err)
-
- coll, err = queries.GetCollectorByClientID(ctx, clientId)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivecollector{
- Clientid: clientId,
- Mincleanversion: 0,
- Mintextversion: 0,
- Activeversion: 1,
- Latestversion: 1,
- }, coll)
-
- err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
- Clientid: clientId,
- Queryid: jsonId,
- Addedversion: 1,
- Name: "example_key",
- })
- require.NoError(t, err)
-
- err = queries.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{
- Clientid: clientId,
- Addedversion: 1,
- Versionid: minTextVersion,
- })
- require.NoError(t, err)
-
- coll, err = queries.GetCollectorByClientID(ctx, clientId)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivecollector{
- Clientid: clientId,
- Mincleanversion: 0,
- Mintextversion: minTextVersion,
- Activeversion: 1,
- Latestversion: 1,
- Fields: []byte(fmt.Sprintf("{\"example_key\": \"%s\"}", jsonId.String())),
- }, coll)
-
- err = queries.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{
- Clientid: clientId,
- Addedversion: 1,
- Versionid: minCleanVersion,
- })
- require.NoError(t, err)
-
- coll, err = queries.GetCollectorByClientID(ctx, clientId)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivecollector{
- Clientid: clientId,
- Mincleanversion: minCleanVersion,
- Mintextversion: minTextVersion,
- Activeversion: 1,
- Latestversion: 1,
- Fields: []byte(fmt.Sprintf("{\"example_key\": \"%s\"}", jsonId.String())),
- }, coll)
-
- qs, err := queries.ListCollectorQueries(ctx, clientId)
- require.NoError(t, err)
- assert.Len(t, qs, 2)
- jsonV := int32(1)
- contextV := int32(0)
- assert.ElementsMatch(t, []*repository.ListCollectorQueriesRow{
- {
- Clientid: &clientId,
- Queryid: &jsonId,
- Queryversion: &jsonV,
- Querytype: repository.NullQuerytype{
- Querytype: repository.QuerytypeJsonExtractor,
- Valid: true,
- },
- Requiredids: []uuid.UUID{contextId},
- },
- {
- Clientid: &clientId,
- Queryid: &contextId,
- Queryversion: &contextV,
- Querytype: repository.NullQuerytype{
- Querytype: repository.QuerytypeContextFull,
- Valid: true,
- },
- Requiredids: []uuid.UUID{},
- },
- }, qs)
-
- version, err = queries.AddLatestCollectorVersion(ctx, clientId)
- require.NoError(t, err)
- assert.Equal(t, int32(2), version)
-
- err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
- Clientid: clientId,
- Queryid: jsonId,
- Removedversion: &version,
- })
- require.NoError(t, err)
-
- err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
- Versionid: 2,
- Clientid: clientId,
- })
- require.NoError(t, err)
-
- err = queries.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{
- Clientid: clientId,
- Versionid: minTextVersion + 1,
- Addedversion: version,
- })
- require.NoError(t, err)
-
- coll, err = queries.GetCollectorByClientID(ctx, clientId)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivecollector{
- Clientid: clientId,
- Mincleanversion: minCleanVersion,
- Mintextversion: minTextVersion + 1,
- Activeversion: 2,
- Latestversion: 2,
- Fields: nil,
- }, coll)
-
- err = queries.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{
- Clientid: clientId,
- Versionid: minCleanVersion + 1,
- Addedversion: version,
- })
- require.NoError(t, err)
-
- coll, err = queries.GetCollectorByClientID(ctx, clientId)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivecollector{
- Clientid: clientId,
- Mincleanversion: minCleanVersion + 1,
- Mintextversion: minTextVersion + 1,
- Activeversion: 2,
- Latestversion: 2,
- Fields: nil,
- }, coll)
-}
-
-func BenchmarkListCollectorQueries(b *testing.B) {
- ctx := b.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(b, cfg)
-
- queries := cfg.GetDBQueries()
-
- clientId := "EXAMPLE"
- err := queries.CreateClient(ctx, &repository.CreateClientParams{
- Name: "example_client",
- Clientid: clientId,
- })
- require.NoError(b, err)
-
- b.ResetTimer()
-
- for b.Loop() {
- _, _ = queries.ListCollectorQueries(ctx, clientId)
- }
-}
-
-func BenchmarkGetCollectorByClientID(b *testing.B) {
- ctx := b.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(b, cfg)
-
- queries := cfg.GetDBQueries()
-
- clientId := "EXAMPLE"
- err := queries.CreateClient(ctx, &repository.CreateClientParams{
- Name: "example_client",
- Clientid: clientId,
- })
- require.NoError(b, err)
-
- b.ResetTimer()
- for b.Loop() {
- _, _ = queries.GetCollectorByClientID(ctx, clientId)
- }
-}
diff --git a/internal/database/repository/document.sql.go b/internal/database/repository/document.sql.go
index d2207133..33708760 100644
--- a/internal/database/repository/document.sql.go
+++ b/internal/database/repository/document.sql.go
@@ -167,74 +167,32 @@ func (q *Queries) GetDocumentEntry(ctx context.Context, documentid uuid.UUID) (*
}
const getDocumentExternal = `-- name: GetDocumentExternal :one
-WITH
-docs AS (
- SELECT id, hash, clientId
- FROM documents
- WHERE id = $1
-),
-namedResults AS (
- SELECT
- q.name,
- dd.id,
- d.value
- FROM currentCollectorQueries AS q
- JOIN docs as dd on dd.clientId = q.clientId
- LEFT JOIN listValidDocumentResults($1) AS d
- ON d.queryId = q.queryId and q.name is not null
-)
SELECT
- d.id,
- d.clientId,
- d.hash,
- COALESCE(jsonb_object_agg(r.name, r.value) FILTER (WHERE r.name IS NOT NULL), '{}') AS fields
-FROM docs AS d
-LEFT JOIN namedResults AS r ON d.id = r.id
-GROUP BY d.id, d.clientId, d.hash
+ id,
+ clientId,
+ hash
+FROM documents
+WHERE id = $1
`
type GetDocumentExternalRow struct {
ID uuid.UUID `db:"id"`
Clientid string `db:"clientid"`
Hash string `db:"hash"`
- Fields []byte `db:"fields"`
}
-// GetDocumentExternal
+// Query functionality has been removed. See remove_query_plan.md for details.
//
-// WITH
-// docs AS (
-// SELECT id, hash, clientId
-// FROM documents
-// WHERE id = $1
-// ),
-// namedResults AS (
-// SELECT
-// q.name,
-// dd.id,
-// d.value
-// FROM currentCollectorQueries AS q
-// JOIN docs as dd on dd.clientId = q.clientId
-// LEFT JOIN listValidDocumentResults($1) AS d
-// ON d.queryId = q.queryId and q.name is not null
-// )
// SELECT
-// d.id,
-// d.clientId,
-// d.hash,
-// COALESCE(jsonb_object_agg(r.name, r.value) FILTER (WHERE r.name IS NOT NULL), '{}') AS fields
-// FROM docs AS d
-// LEFT JOIN namedResults AS r ON d.id = r.id
-// GROUP BY d.id, d.clientId, d.hash
+// id,
+// clientId,
+// hash
+// FROM documents
+// WHERE id = $1
func (q *Queries) GetDocumentExternal(ctx context.Context, documentid uuid.UUID) (*GetDocumentExternalRow, error) {
row := q.db.QueryRow(ctx, getDocumentExternal, documentid)
var i GetDocumentExternalRow
- err := row.Scan(
- &i.ID,
- &i.Clientid,
- &i.Hash,
- &i.Fields,
- )
+ err := row.Scan(&i.ID, &i.Clientid, &i.Hash)
return &i, err
}
diff --git a/internal/database/repository/document_test.go b/internal/database/repository/document_test.go
index 18ab2732..e3009f9d 100644
--- a/internal/database/repository/document_test.go
+++ b/internal/database/repository/document_test.go
@@ -218,7 +218,6 @@ func TestDocument(t *testing.T) {
ID: id,
Clientid: clientId,
Hash: hash,
- Fields: []byte("{}"),
}, docext)
})
t.Run("doc id by hash", func(t *testing.T) {
diff --git a/internal/database/repository/models.go b/internal/database/repository/models.go
index d0b0c27c..f714953d 100644
--- a/internal/database/repository/models.go
+++ b/internal/database/repository/models.go
@@ -181,57 +181,6 @@ func (e Cleanmimetype) Valid() bool {
return false
}
-type Querytype string
-
-const (
- QuerytypeContextFull Querytype = "context_full"
- QuerytypeJsonExtractor Querytype = "json_extractor"
-)
-
-func (e *Querytype) Scan(src interface{}) error {
- switch s := src.(type) {
- case []byte:
- *e = Querytype(s)
- case string:
- *e = Querytype(s)
- default:
- return fmt.Errorf("unsupported scan type for Querytype: %T", src)
- }
- return nil
-}
-
-type NullQuerytype struct {
- Querytype Querytype
- Valid bool // Valid is true if Querytype is not NULL
-}
-
-// Scan implements the Scanner interface.
-func (ns *NullQuerytype) Scan(value interface{}) error {
- if value == nil {
- ns.Querytype, ns.Valid = "", false
- return nil
- }
- ns.Valid = true
- return ns.Querytype.Scan(value)
-}
-
-// Value implements the driver Valuer interface.
-func (ns NullQuerytype) Value() (driver.Value, error) {
- if !ns.Valid {
- return nil, nil
- }
- return string(ns.Querytype), nil
-}
-
-func (e Querytype) Valid() bool {
- switch e {
- case QuerytypeContextFull,
- QuerytypeJsonExtractor:
- return true
- }
- return false
-}
-
type BatchUpload struct {
ID uuid.UUID `db:"id"`
ClientID string `db:"client_id"`
@@ -293,23 +242,6 @@ type Collectormintextversion struct {
Removedversion *int32 `db:"removedversion"`
}
-type Collectorquery struct {
- ID uuid.UUID `db:"id"`
- Clientid string `db:"clientid"`
- Name string `db:"name"`
- Queryid uuid.UUID `db:"queryid"`
- Addedversion int32 `db:"addedversion"`
- Removedversion *int32 `db:"removedversion"`
-}
-
-type Collectorquerydependencytree struct {
- Clientid string `db:"clientid"`
- Queryid *uuid.UUID `db:"queryid"`
- Querytype Querytype `db:"querytype"`
- Queryversion int32 `db:"queryversion"`
- Requiredids []uuid.UUID `db:"requiredids"`
-}
-
type Collectorversion struct {
Clientid string `db:"clientid"`
ID int32 `db:"id"`
@@ -343,17 +275,6 @@ type Currentcollectormintextversion struct {
Mintextversion int64 `db:"mintextversion"`
}
-type Currentcollectorqueriesjsonagg struct {
- Clientid string `db:"clientid"`
- Fields []byte `db:"fields"`
-}
-
-type Currentcollectorquery struct {
- Clientid string `db:"clientid"`
- Name *string `db:"name"`
- Queryid *uuid.UUID `db:"queryid"`
-}
-
type Currentfieldextraction struct {
ID uuid.UUID `db:"id"`
Documentid uuid.UUID `db:"documentid"`
@@ -659,16 +580,6 @@ type Fullactivecollector struct {
Mintextversion int64 `db:"mintextversion"`
Activeversion int32 `db:"activeversion"`
Latestversion int32 `db:"latestversion"`
- Fields []byte `db:"fields"`
-}
-
-type Fullactivequery struct {
- Queryid uuid.UUID `db:"queryid"`
- Querytype Querytype `db:"querytype"`
- Activeversion int32 `db:"activeversion"`
- Latestversion int32 `db:"latestversion"`
- Config []byte `db:"config"`
- Requiredids []uuid.UUID `db:"requiredids"`
}
type Fullclient struct {
@@ -681,79 +592,3 @@ type Label struct {
Label string `db:"label"`
Description string `db:"description"`
}
-
-type Query struct {
- Queryid uuid.UUID `db:"queryid"`
- Querytype Querytype `db:"querytype"`
-}
-
-type Queryactivedependency struct {
- ID uuid.UUID `db:"id"`
- Requiredqueryid *uuid.UUID `db:"requiredqueryid"`
-}
-
-type Queryactiveversion struct {
- Activeversionentryid uuid.UUID `db:"activeversionentryid"`
- Queryid uuid.UUID `db:"queryid"`
- Versionid int32 `db:"versionid"`
-}
-
-type Queryconfig struct {
- Configid uuid.UUID `db:"configid"`
- Queryid uuid.UUID `db:"queryid"`
- Config []byte `db:"config"`
- Addedversion int32 `db:"addedversion"`
- Removedversion *int32 `db:"removedversion"`
-}
-
-type Querycurrentactiveversion struct {
- Queryid uuid.UUID `db:"queryid"`
- Activeversion int32 `db:"activeversion"`
-}
-
-type Querycurrentconfig struct {
- Queryid uuid.UUID `db:"queryid"`
- Config []byte `db:"config"`
-}
-
-type Querycurrentrequiredid struct {
- Queryid uuid.UUID `db:"queryid"`
- Requiredqueryid *uuid.UUID `db:"requiredqueryid"`
-}
-
-type Querycurrentrequiredidsagg struct {
- Queryid uuid.UUID `db:"queryid"`
- Requiredids []uuid.UUID `db:"requiredids"`
-}
-
-type Querylatestversion struct {
- Queryid uuid.UUID `db:"queryid"`
- Latestversion int32 `db:"latestversion"`
-}
-
-type Queryversion struct {
- Queryid uuid.UUID `db:"queryid"`
- Versionid int32 `db:"versionid"`
- Addedat pgtype.Timestamp `db:"addedat"`
-}
-
-type Requiredquery struct {
- Requiredqueryentryid uuid.UUID `db:"requiredqueryentryid"`
- Queryid uuid.UUID `db:"queryid"`
- Requiredqueryid uuid.UUID `db:"requiredqueryid"`
- Addedversion int32 `db:"addedversion"`
- Removedversion *int32 `db:"removedversion"`
-}
-
-type Result struct {
- ID uuid.UUID `db:"id"`
- Textentryid uuid.UUID `db:"textentryid"`
- Queryid uuid.UUID `db:"queryid"`
- Value string `db:"value"`
- Queryversion int32 `db:"queryversion"`
-}
-
-type Resultdependency struct {
- Resultid uuid.UUID `db:"resultid"`
- Requiredresultid uuid.UUID `db:"requiredresultid"`
-}
diff --git a/internal/database/repository/query.sql.go b/internal/database/repository/query.sql.go
deleted file mode 100644
index 345d96b1..00000000
--- a/internal/database/repository/query.sql.go
+++ /dev/null
@@ -1,425 +0,0 @@
-// Code generated by sqlc. DO NOT EDIT.
-// versions:
-// sqlc v1.27.0
-// source: query.sql
-
-package repository
-
-import (
- "context"
-
- "github.com/google/uuid"
-)
-
-const addActiveQueryVersion = `-- name: AddActiveQueryVersion :exec
-INSERT INTO queryActiveVersions (queryId, versionId) VALUES ($1, $2)
-`
-
-type AddActiveQueryVersionParams struct {
- Queryid uuid.UUID `db:"queryid"`
- Versionid int32 `db:"versionid"`
-}
-
-// AddActiveQueryVersion
-//
-// INSERT INTO queryActiveVersions (queryId, versionId) VALUES ($1, $2)
-func (q *Queries) AddActiveQueryVersion(ctx context.Context, arg *AddActiveQueryVersionParams) error {
- _, err := q.db.Exec(ctx, addActiveQueryVersion, arg.Queryid, arg.Versionid)
- return err
-}
-
-const addLatestQueryVersion = `-- name: AddLatestQueryVersion :one
-INSERT INTO queryVersions (queryId) VALUES ($1) RETURNING versionId
-`
-
-// AddLatestQueryVersion
-//
-// INSERT INTO queryVersions (queryId) VALUES ($1) RETURNING versionId
-func (q *Queries) AddLatestQueryVersion(ctx context.Context, queryid uuid.UUID) (int32, error) {
- row := q.db.QueryRow(ctx, addLatestQueryVersion, queryid)
- var versionid int32
- err := row.Scan(&versionid)
- return versionid, err
-}
-
-const addRequiredQuery = `-- name: AddRequiredQuery :exec
-INSERT INTO requiredQueries (queryId, requiredQueryId, addedVersion) VALUES ($1, $2, $3)
-`
-
-type AddRequiredQueryParams struct {
- Queryid uuid.UUID `db:"queryid"`
- Requiredqueryid uuid.UUID `db:"requiredqueryid"`
- Addedversion int32 `db:"addedversion"`
-}
-
-// AddRequiredQuery
-//
-// INSERT INTO requiredQueries (queryId, requiredQueryId, addedVersion) VALUES ($1, $2, $3)
-func (q *Queries) AddRequiredQuery(ctx context.Context, arg *AddRequiredQueryParams) error {
- _, err := q.db.Exec(ctx, addRequiredQuery, arg.Queryid, arg.Requiredqueryid, arg.Addedversion)
- return err
-}
-
-const allQueriesExist = `-- name: AllQueriesExist :one
-SELECT COUNT(*) = COUNT(DISTINCT queryId) AS all_exist
- FROM unnest($1::uuid[]) AS input_id
- LEFT JOIN queries ON input_id = queries.queryId
-`
-
-// AllQueriesExist
-//
-// SELECT COUNT(*) = COUNT(DISTINCT queryId) AS all_exist
-// FROM unnest($1::uuid[]) AS input_id
-// LEFT JOIN queries ON input_id = queries.queryId
-func (q *Queries) AllQueriesExist(ctx context.Context, dollar_1 []uuid.UUID) (bool, error) {
- row := q.db.QueryRow(ctx, allQueriesExist, dollar_1)
- var all_exist bool
- err := row.Scan(&all_exist)
- return all_exist, err
-}
-
-const createQuery = `-- name: CreateQuery :one
-INSERT INTO queries (queryType) VALUES ($1) RETURNING queryId
-`
-
-// CreateQuery
-//
-// INSERT INTO queries (queryType) VALUES ($1) RETURNING queryId
-func (q *Queries) CreateQuery(ctx context.Context, querytype Querytype) (uuid.UUID, error) {
- row := q.db.QueryRow(ctx, createQuery, querytype)
- var queryid uuid.UUID
- err := row.Scan(&queryid)
- return queryid, err
-}
-
-const getActiveQueryConfig = `-- name: GetActiveQueryConfig :one
-SELECT config FROM queryCurrentConfigs where queryId = $1
-`
-
-// GetActiveQueryConfig
-//
-// SELECT config FROM queryCurrentConfigs where queryId = $1
-func (q *Queries) GetActiveQueryConfig(ctx context.Context, queryid uuid.UUID) ([]byte, error) {
- row := q.db.QueryRow(ctx, getActiveQueryConfig, queryid)
- var config []byte
- err := row.Scan(&config)
- return config, err
-}
-
-const getQuery = `-- name: GetQuery :one
-SELECT queryid, querytype, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE queryId = $1
-`
-
-// GetQuery
-//
-// SELECT queryid, querytype, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE queryId = $1
-func (q *Queries) GetQuery(ctx context.Context, queryid uuid.UUID) (*Fullactivequery, error) {
- row := q.db.QueryRow(ctx, getQuery, queryid)
- var i Fullactivequery
- err := row.Scan(
- &i.Queryid,
- &i.Querytype,
- &i.Activeversion,
- &i.Latestversion,
- &i.Config,
- &i.Requiredids,
- )
- return &i, err
-}
-
-const getQueryWithVersion = `-- name: GetQueryWithVersion :one
-WITH query as (
- SELECT queryId, queryType FROM queries WHERE queryId = $1
-),
-config as (
-SELECT c.queryId, c.config
- FROM query AS q
- LEFT JOIN queryConfigs AS c ON q.queryId = c.queryId
- and isInVersion($2, c.addedVersion, c.removedVersion)
-),
-requiredIds as (
- SELECT r.queryId,
- coalesce(
- ARRAY_AGG(DISTINCT r.requiredQueryId)
- FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
- array[]::uuid[]
- )::uuid[] as requiredIds
- FROM query AS q
- LEFT JOIN requiredQueries AS r ON q.queryId = r.queryId
- and isInVersion($2, r.addedVersion, r.removedVersion)
- GROUP BY r.queryId
-)
-SELECT DISTINCT q.queryId, q.queryType,
- av.activeVersion,
- lv.latestVersion, c.config,
- r.requiredIds
- FROM query AS q
- JOIN queryCurrentActiveVersions as av on q.queryId = av.queryId
- JOIN queryLatestVersions as lv on lv.queryId = q.queryId
- LEFT JOIN config AS c ON q.queryId = c.queryId
- LEFT JOIN requiredIds AS r ON q.queryId = r.queryId
-`
-
-type GetQueryWithVersionParams struct {
- ID *uuid.UUID `db:"id"`
- Version *int32 `db:"version"`
-}
-
-type GetQueryWithVersionRow struct {
- Queryid uuid.UUID `db:"queryid"`
- Querytype Querytype `db:"querytype"`
- Activeversion int32 `db:"activeversion"`
- Latestversion int32 `db:"latestversion"`
- Config []byte `db:"config"`
- Requiredids []uuid.UUID `db:"requiredids"`
-}
-
-// GetQueryWithVersion
-//
-// WITH query as (
-// SELECT queryId, queryType FROM queries WHERE queryId = $1
-// ),
-// config as (
-// SELECT c.queryId, c.config
-// FROM query AS q
-// LEFT JOIN queryConfigs AS c ON q.queryId = c.queryId
-// and isInVersion($2, c.addedVersion, c.removedVersion)
-// ),
-// requiredIds as (
-// SELECT r.queryId,
-// coalesce(
-// ARRAY_AGG(DISTINCT r.requiredQueryId)
-// FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
-// array[]::uuid[]
-// )::uuid[] as requiredIds
-// FROM query AS q
-// LEFT JOIN requiredQueries AS r ON q.queryId = r.queryId
-// and isInVersion($2, r.addedVersion, r.removedVersion)
-// GROUP BY r.queryId
-// )
-// SELECT DISTINCT q.queryId, q.queryType,
-// av.activeVersion,
-// lv.latestVersion, c.config,
-// r.requiredIds
-// FROM query AS q
-// JOIN queryCurrentActiveVersions as av on q.queryId = av.queryId
-// JOIN queryLatestVersions as lv on lv.queryId = q.queryId
-// LEFT JOIN config AS c ON q.queryId = c.queryId
-// LEFT JOIN requiredIds AS r ON q.queryId = r.queryId
-func (q *Queries) GetQueryWithVersion(ctx context.Context, arg *GetQueryWithVersionParams) (*GetQueryWithVersionRow, error) {
- row := q.db.QueryRow(ctx, getQueryWithVersion, arg.ID, arg.Version)
- var i GetQueryWithVersionRow
- err := row.Scan(
- &i.Queryid,
- &i.Querytype,
- &i.Activeversion,
- &i.Latestversion,
- &i.Config,
- &i.Requiredids,
- )
- return &i, err
-}
-
-const isQueryInDependencyTree = `-- name: IsQueryInDependencyTree :one
-SELECT EXISTS (
- SELECT 1 FROM queryActiveDependencies
- WHERE id = any($1)
- and requiredQueryId = $2
- or $2 = any($1)
-)
-`
-
-type IsQueryInDependencyTreeParams struct {
- Requiredqueryids []uuid.UUID `db:"requiredqueryids"`
- Queryid *uuid.UUID `db:"queryid"`
-}
-
-// IsQueryInDependencyTree
-//
-// SELECT EXISTS (
-// SELECT 1 FROM queryActiveDependencies
-// WHERE id = any($1)
-// and requiredQueryId = $2
-// or $2 = any($1)
-// )
-func (q *Queries) IsQueryInDependencyTree(ctx context.Context, arg *IsQueryInDependencyTreeParams) (bool, error) {
- row := q.db.QueryRow(ctx, isQueryInDependencyTree, arg.Requiredqueryids, arg.Queryid)
- var exists bool
- err := row.Scan(&exists)
- return exists, err
-}
-
-const listQueries = `-- name: ListQueries :many
-SELECT queryid, querytype, activeversion, latestversion, config, requiredids FROM fullActiveQueries
-`
-
-// ListQueries
-//
-// SELECT queryid, querytype, activeversion, latestversion, config, requiredids FROM fullActiveQueries
-func (q *Queries) ListQueries(ctx context.Context) ([]*Fullactivequery, error) {
- rows, err := q.db.Query(ctx, listQueries)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
- items := []*Fullactivequery{}
- for rows.Next() {
- var i Fullactivequery
- if err := rows.Scan(
- &i.Queryid,
- &i.Querytype,
- &i.Activeversion,
- &i.Latestversion,
- &i.Config,
- &i.Requiredids,
- ); err != nil {
- return nil, err
- }
- items = append(items, &i)
- }
- if err := rows.Err(); err != nil {
- return nil, err
- }
- return items, nil
-}
-
-const listQueriesById = `-- name: ListQueriesById :many
-SELECT queryid, querytype, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE queryId = any($1)
-`
-
-// ListQueriesById
-//
-// SELECT queryid, querytype, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE queryId = any($1)
-func (q *Queries) ListQueriesById(ctx context.Context, queryid []uuid.UUID) ([]*Fullactivequery, error) {
- rows, err := q.db.Query(ctx, listQueriesById, queryid)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
- items := []*Fullactivequery{}
- for rows.Next() {
- var i Fullactivequery
- if err := rows.Scan(
- &i.Queryid,
- &i.Querytype,
- &i.Activeversion,
- &i.Latestversion,
- &i.Config,
- &i.Requiredids,
- ); err != nil {
- return nil, err
- }
- items = append(items, &i)
- }
- if err := rows.Err(); err != nil {
- return nil, err
- }
- return items, nil
-}
-
-const listQueryClientIDs = `-- name: ListQueryClientIDs :many
-SELECT clientId FROM collectorQueryDependencyTree WHERE queryId = $1
-`
-
-// ListQueryClientIDs
-//
-// SELECT clientId FROM collectorQueryDependencyTree WHERE queryId = $1
-func (q *Queries) ListQueryClientIDs(ctx context.Context, queryid *uuid.UUID) ([]string, error) {
- rows, err := q.db.Query(ctx, listQueryClientIDs, queryid)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
- items := []string{}
- for rows.Next() {
- var clientid string
- if err := rows.Scan(&clientid); err != nil {
- return nil, err
- }
- items = append(items, clientid)
- }
- if err := rows.Err(); err != nil {
- return nil, err
- }
- return items, nil
-}
-
-const listQueryDirectDependentsByDocumentID = `-- name: ListQueryDirectDependentsByDocumentID :many
-WITH doc AS (
- SELECT clientId FROM documents WHERE id = $2
-)
-SELECT dt.queryId
-FROM doc,
- LATERAL collectorQueryDependencyTreeByClient(doc.clientId) AS dt
-WHERE $1 = ANY(dt.requiredIds)
-`
-
-type ListQueryDirectDependentsByDocumentIDParams struct {
- Queryid *uuid.UUID `db:"queryid"`
- Documentid *uuid.UUID `db:"documentid"`
-}
-
-// ListQueryDirectDependentsByDocumentID
-//
-// WITH doc AS (
-// SELECT clientId FROM documents WHERE id = $2
-// )
-// SELECT dt.queryId
-// FROM doc,
-// LATERAL collectorQueryDependencyTreeByClient(doc.clientId) AS dt
-// WHERE $1 = ANY(dt.requiredIds)
-func (q *Queries) ListQueryDirectDependentsByDocumentID(ctx context.Context, arg *ListQueryDirectDependentsByDocumentIDParams) ([]*uuid.UUID, error) {
- rows, err := q.db.Query(ctx, listQueryDirectDependentsByDocumentID, arg.Queryid, arg.Documentid)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
- items := []*uuid.UUID{}
- for rows.Next() {
- var queryid *uuid.UUID
- if err := rows.Scan(&queryid); err != nil {
- return nil, err
- }
- items = append(items, queryid)
- }
- if err := rows.Err(); err != nil {
- return nil, err
- }
- return items, nil
-}
-
-const removeRequiredQuery = `-- name: RemoveRequiredQuery :exec
-UPDATE requiredQueries SET removedVersion = $1 WHERE requiredQueryId = $2 and queryId = $3 and removedVersion is null
-`
-
-type RemoveRequiredQueryParams struct {
- Removedversion *int32 `db:"removedversion"`
- Requiredqueryid uuid.UUID `db:"requiredqueryid"`
- Queryid uuid.UUID `db:"queryid"`
-}
-
-// RemoveRequiredQuery
-//
-// UPDATE requiredQueries SET removedVersion = $1 WHERE requiredQueryId = $2 and queryId = $3 and removedVersion is null
-func (q *Queries) RemoveRequiredQuery(ctx context.Context, arg *RemoveRequiredQueryParams) error {
- _, err := q.db.Exec(ctx, removeRequiredQuery, arg.Removedversion, arg.Requiredqueryid, arg.Queryid)
- return err
-}
-
-const setQueryConfig = `-- name: SetQueryConfig :exec
-INSERT INTO queryConfigs (queryId, config, addedVersion) VALUES ($1, $2, $3)
-`
-
-type SetQueryConfigParams struct {
- Queryid uuid.UUID `db:"queryid"`
- Config []byte `db:"config"`
- Addedversion int32 `db:"addedversion"`
-}
-
-// SetQueryConfig
-//
-// INSERT INTO queryConfigs (queryId, config, addedVersion) VALUES ($1, $2, $3)
-func (q *Queries) SetQueryConfig(ctx context.Context, arg *SetQueryConfigParams) error {
- _, err := q.db.Exec(ctx, setQueryConfig, arg.Queryid, arg.Config, arg.Addedversion)
- return err
-}
diff --git a/internal/database/repository/query_test.go b/internal/database/repository/query_test.go
deleted file mode 100644
index 1885fefa..00000000
--- a/internal/database/repository/query_test.go
+++ /dev/null
@@ -1,622 +0,0 @@
-package repository_test
-
-import (
- "testing"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/test"
-
- "github.com/google/uuid"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestQueries(t *testing.T) {
- t.Parallel()
- if testing.Short() {
- t.SkipNow()
- }
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
- require.NoError(t, err)
-
- contextQuery, err := queries.GetQuery(ctx, contextQueryID)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivequery{
- Queryid: contextQueryID,
- Querytype: repository.QuerytypeContextFull,
- Activeversion: 0,
- Latestversion: 0,
- Config: nil,
- Requiredids: []uuid.UUID{},
- }, contextQuery)
-
- ctxVersion, err := queries.AddLatestQueryVersion(ctx, contextQueryID)
- require.NoError(t, err)
- assert.Equal(t, int32(1), ctxVersion)
-
- contextQuery, err = queries.GetQuery(ctx, contextQueryID)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivequery{
- Queryid: contextQueryID,
- Querytype: repository.QuerytypeContextFull,
- Activeversion: 0,
- Latestversion: 1,
- Config: nil,
- Requiredids: []uuid.UUID{},
- }, contextQuery)
-
- jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
- require.NoError(t, err)
-
- jsonQuery, err := queries.GetQuery(ctx, jsonQueryID)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivequery{
- Queryid: jsonQueryID,
- Querytype: repository.QuerytypeJsonExtractor,
- Activeversion: 0,
- Latestversion: 0,
- Config: nil,
- Requiredids: []uuid.UUID{},
- }, jsonQuery)
-
- jsonVersion, err := queries.AddLatestQueryVersion(ctx, jsonQueryID)
- require.NoError(t, err)
- assert.Equal(t, int32(1), jsonVersion)
-
- jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivequery{
- Queryid: jsonQueryID,
- Querytype: repository.QuerytypeJsonExtractor,
- Activeversion: 0,
- Latestversion: 1,
- Config: nil,
- Requiredids: []uuid.UUID{},
- }, jsonQuery)
-
- err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
- Versionid: 1,
- Queryid: jsonQueryID,
- })
- require.NoError(t, err)
-
- jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivequery{
- Queryid: jsonQueryID,
- Querytype: repository.QuerytypeJsonExtractor,
- Activeversion: 1,
- Latestversion: 1,
- Config: nil,
- Requiredids: []uuid.UUID{},
- }, jsonQuery)
-
- jsonConfig := []byte("{\"path\": \"example_path\"}")
-
- err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
- Queryid: jsonQueryID,
- Requiredqueryid: contextQueryID,
- Addedversion: 1,
- })
- require.NoError(t, err)
-
- jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivequery{
- Queryid: jsonQueryID,
- Querytype: repository.QuerytypeJsonExtractor,
- Activeversion: 1,
- Latestversion: 1,
- Config: nil,
- Requiredids: []uuid.UUID{contextQueryID},
- }, jsonQuery)
-
- jsonVersion, err = queries.AddLatestQueryVersion(ctx, jsonQueryID)
- require.NoError(t, err)
- assert.Equal(t, int32(2), jsonVersion)
-
- jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivequery{
- Queryid: jsonQueryID,
- Querytype: repository.QuerytypeJsonExtractor,
- Activeversion: 1,
- Latestversion: 2,
- Config: nil,
- Requiredids: []uuid.UUID{contextQueryID},
- }, jsonQuery)
-
- removeV := int32(2)
- err = queries.RemoveRequiredQuery(ctx, &repository.RemoveRequiredQueryParams{
- Queryid: jsonQueryID,
- Requiredqueryid: contextQueryID,
- Removedversion: &removeV,
- })
- require.NoError(t, err)
-
- err = queries.SetQueryConfig(ctx, &repository.SetQueryConfigParams{
- Queryid: jsonQueryID,
- Config: jsonConfig,
- Addedversion: 1,
- })
- require.NoError(t, err)
-
- jsonQueryConfig, err := queries.GetActiveQueryConfig(ctx, jsonQueryID)
- require.NoError(t, err)
- assert.Equal(t, jsonConfig, jsonQueryConfig)
-
- err = queries.SetQueryConfig(ctx, &repository.SetQueryConfigParams{
- Queryid: jsonQueryID,
- Config: []byte(`{"second":"key"}`),
- Addedversion: 2,
- })
- require.NoError(t, err)
-
- jsonQueryConfig, err = queries.GetActiveQueryConfig(ctx, jsonQueryID)
- require.NoError(t, err)
- assert.Equal(t, jsonConfig, jsonQueryConfig)
-
- err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
- Queryid: jsonQueryID,
- Versionid: 2,
- })
- require.NoError(t, err)
-
- jsonQueryConfig, err = queries.GetActiveQueryConfig(ctx, jsonQueryID)
- require.NoError(t, err)
- assert.Equal(t, []byte(`{"second": "key"}`), jsonQueryConfig)
-
- jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.Fullactivequery{
- Queryid: jsonQueryID,
- Querytype: repository.QuerytypeJsonExtractor,
- Activeversion: 2,
- Latestversion: 2,
- Config: jsonQueryConfig,
- Requiredids: []uuid.UUID{},
- }, jsonQuery)
-
- v := int32(1)
- versionedQuery, err := queries.GetQueryWithVersion(ctx, &repository.GetQueryWithVersionParams{
- ID: &jsonQueryID,
- Version: &v,
- })
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.GetQueryWithVersionRow{
- Queryid: jsonQueryID,
- Querytype: repository.QuerytypeJsonExtractor,
- Activeversion: 2,
- Latestversion: 2,
- Config: jsonConfig,
- Requiredids: []uuid.UUID{contextQueryID},
- }, versionedQuery)
-
- all_exist, err := queries.AllQueriesExist(ctx, []uuid.UUID{})
- require.NoError(t, err)
- assert.True(t, all_exist)
-
- all_exist, err = queries.AllQueriesExist(ctx, []uuid.UUID{uuid.New()})
- require.NoError(t, err)
- assert.False(t, all_exist)
-
- all_exist, err = queries.AllQueriesExist(ctx, []uuid.UUID{jsonQueryID})
- require.NoError(t, err)
- assert.True(t, all_exist)
-
- all_exist, err = queries.AllQueriesExist(ctx, []uuid.UUID{jsonQueryID, contextQueryID})
- require.NoError(t, err)
- assert.True(t, all_exist)
-
- all_exist, err = queries.AllQueriesExist(ctx, []uuid.UUID{jsonQueryID, uuid.New()})
- require.NoError(t, err)
- assert.False(t, all_exist)
-}
-
-func TestQueryDependencyTree(t *testing.T) {
- t.Parallel()
- if testing.Short() {
- t.SkipNow()
- }
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- clientID := "EXAMPLE"
- err := queries.CreateClient(ctx, &repository.CreateClientParams{
- Name: "example_client",
- Clientid: clientID,
- })
- require.NoError(t, err)
- version, err := queries.AddLatestCollectorVersion(ctx, clientID)
- require.NoError(t, err)
- err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
- Clientid: clientID,
- Versionid: version,
- })
- require.NoError(t, err)
- docID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
- Clientid: clientID,
- Hash: "sample",
- BatchID: nil,
- })
- require.NoError(t, err)
-
- contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
- require.NoError(t, err)
- _, err = queries.AddLatestQueryVersion(ctx, contextQueryID)
- require.NoError(t, err)
- err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
- Queryid: contextQueryID,
- Versionid: 1,
- })
- require.NoError(t, err)
-
- dependents, err := queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &contextQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []uuid.UUID{}, dependents)
-
- jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
- require.NoError(t, err)
- _, err = queries.AddLatestQueryVersion(ctx, jsonQueryID)
- require.NoError(t, err)
- err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
- Queryid: jsonQueryID,
- Versionid: 1,
- })
- require.NoError(t, err)
-
- dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &jsonQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []uuid.UUID{}, dependents)
-
- err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
- Queryid: jsonQueryID,
- Requiredqueryid: contextQueryID,
- Addedversion: 1,
- })
- require.NoError(t, err)
-
- dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &jsonQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []uuid.UUID{}, dependents)
- dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &contextQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []uuid.UUID{}, dependents)
-
- err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
- Clientid: clientID,
- Name: "example_name'",
- Queryid: jsonQueryID,
- Addedversion: 1,
- })
- require.NoError(t, err)
-
- dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &jsonQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []*uuid.UUID{}, dependents)
- dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &contextQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, dependents)
-
- secondJsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
- require.NoError(t, err)
- _, err = queries.AddLatestQueryVersion(ctx, secondJsonQueryID)
- require.NoError(t, err)
- err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
- Queryid: secondJsonQueryID,
- Versionid: 1,
- })
- require.NoError(t, err)
-
- dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &jsonQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []*uuid.UUID{}, dependents)
- dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &contextQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, dependents)
-
- err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
- Queryid: secondJsonQueryID,
- Requiredqueryid: jsonQueryID,
- Addedversion: 1,
- })
- require.NoError(t, err)
-
- dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &secondJsonQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []*uuid.UUID{}, dependents)
- dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &jsonQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []*uuid.UUID{}, dependents)
- dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &contextQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, dependents)
-
- err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
- Clientid: clientID,
- Name: "example_name'",
- Queryid: secondJsonQueryID,
- Addedversion: 1,
- })
- require.NoError(t, err)
-
- dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &secondJsonQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []*uuid.UUID{}, dependents)
- dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &jsonQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []*uuid.UUID{&secondJsonQueryID}, dependents)
- dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &contextQueryID,
- })
- require.NoError(t, err)
- assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, dependents)
-
- isdependent, err := queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
- Queryid: &jsonQueryID,
- Requiredqueryids: []uuid.UUID{contextQueryID},
- })
- require.NoError(t, err)
- assert.False(t, isdependent)
-
- isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
- Queryid: &jsonQueryID,
- Requiredqueryids: []uuid.UUID{secondJsonQueryID},
- })
- require.NoError(t, err)
- assert.True(t, isdependent)
-
- isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
- Queryid: &jsonQueryID,
- Requiredqueryids: []uuid.UUID{jsonQueryID},
- })
- require.NoError(t, err)
- assert.True(t, isdependent)
-
- isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
- Queryid: &secondJsonQueryID,
- Requiredqueryids: []uuid.UUID{jsonQueryID, contextQueryID},
- })
- require.NoError(t, err)
- assert.False(t, isdependent)
-
- isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
- Queryid: &contextQueryID,
- Requiredqueryids: []uuid.UUID{jsonQueryID, secondJsonQueryID},
- })
- require.NoError(t, err)
- assert.True(t, isdependent)
-}
-
-func TestQueriesList(t *testing.T) {
- t.Parallel()
- if testing.Short() {
- t.SkipNow()
- }
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
- require.NoError(t, err)
-
- jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
- require.NoError(t, err)
-
- qs, err := queries.ListQueries(ctx)
- require.NoError(t, err)
- assert.Len(t, qs, 2)
- assert.ElementsMatch(t, []*repository.Fullactivequery{
- {
- Queryid: jsonQueryID,
- Querytype: repository.QuerytypeJsonExtractor,
- Activeversion: 0,
- Latestversion: 0,
- Config: nil,
- Requiredids: []uuid.UUID{},
- },
- {
- Queryid: contextQueryID,
- Querytype: repository.QuerytypeContextFull,
- Activeversion: 0,
- Latestversion: 0,
- Config: nil,
- Requiredids: []uuid.UUID{},
- },
- }, qs)
-
- qs, err = queries.ListQueriesById(ctx, []uuid.UUID{jsonQueryID})
- require.NoError(t, err)
- assert.Len(t, qs, 1)
- assert.ElementsMatch(t, []*repository.Fullactivequery{
- {
- Queryid: jsonQueryID,
- Querytype: repository.QuerytypeJsonExtractor,
- Activeversion: 0,
- Latestversion: 0,
- Config: nil,
- Requiredids: []uuid.UUID{},
- },
- }, qs)
-}
-
-func TestListQueryClients(t *testing.T) {
- t.Parallel()
- if testing.Short() {
- t.SkipNow()
- }
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- contextID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
- require.NoError(t, err)
-
- clients, err := queries.ListQueryClientIDs(ctx, &contextID)
- require.NoError(t, err)
- assert.ElementsMatch(t, []uuid.UUID{}, clients)
-
- clientOneID := "EXAMPLE"
- err = queries.CreateClient(ctx, &repository.CreateClientParams{
- Name: "example_client",
- Clientid: clientOneID,
- })
- require.NoError(t, err)
- versionOne, err := queries.AddLatestCollectorVersion(ctx, clientOneID)
- require.NoError(t, err)
- err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
- Clientid: clientOneID,
- Versionid: versionOne,
- })
- require.NoError(t, err)
- err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
- Clientid: clientOneID,
- Queryid: contextID,
- Addedversion: 1,
- Name: "example_key",
- })
- require.NoError(t, err)
-
- clients, err = queries.ListQueryClientIDs(ctx, &contextID)
- require.NoError(t, err)
- assert.ElementsMatch(t, []string{clientOneID}, clients)
-
- clientTwoID := "EXAMPLE_DOS"
- err = queries.CreateClient(ctx, &repository.CreateClientParams{
- Name: "example_client_dos",
- Clientid: clientTwoID,
- })
- require.NoError(t, err)
- versionTwo, err := queries.AddLatestCollectorVersion(ctx, clientTwoID)
- require.NoError(t, err)
- err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
- Clientid: clientTwoID,
- Versionid: versionTwo,
- })
- require.NoError(t, err)
- err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
- Clientid: clientTwoID,
- Queryid: contextID,
- Addedversion: 1,
- Name: "example_key",
- })
- require.NoError(t, err)
-
- clients, err = queries.ListQueryClientIDs(ctx, &contextID)
- require.NoError(t, err)
- assert.ElementsMatch(t, []string{clientOneID, clientTwoID}, clients)
-
- jsonID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
- require.NoError(t, err)
- _, err = queries.AddLatestQueryVersion(ctx, jsonID)
- require.NoError(t, err)
- err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
- Queryid: jsonID,
- Requiredqueryid: contextID,
- Addedversion: 1,
- })
- require.NoError(t, err)
- err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
- Clientid: clientOneID,
- Queryid: jsonID,
- Addedversion: 1,
- Name: "example_key",
- })
- require.NoError(t, err)
-
- clients, err = queries.ListQueryClientIDs(ctx, &jsonID)
- require.NoError(t, err)
- assert.ElementsMatch(t, []string{clientOneID}, clients)
-}
-
-func BenchmarkListQueryDirectDependentsByDocId(b *testing.B) {
- ctx := b.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(b, cfg)
-
- queries := cfg.GetDBQueries()
-
- clientID := "EXAMPLE"
- err := queries.CreateClient(ctx, &repository.CreateClientParams{
- Name: "example_client",
- Clientid: clientID,
- })
- require.NoError(b, err)
-
- docID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
- Clientid: clientID,
- Hash: "sample",
- BatchID: nil,
- })
- require.NoError(b, err)
-
- contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
- require.NoError(b, err)
-
- b.ResetTimer()
- for b.Loop() {
- _, _ = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: &docID,
- Queryid: &contextQueryID,
- })
- }
-}
diff --git a/internal/database/repository/querytype_test.go b/internal/database/repository/querytype_test.go
deleted file mode 100644
index aac36288..00000000
--- a/internal/database/repository/querytype_test.go
+++ /dev/null
@@ -1,43 +0,0 @@
-package repository_test
-
-import (
- "fmt"
- "testing"
-
- "queryorchestration/internal/database/repository"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestQueryTypeScan(t *testing.T) {
- qType := repository.Querytype(fmt.Sprint(0))
-
- stringType := "context_full"
- err := qType.Scan(stringType)
- require.NoError(t, err)
-}
-
-func TestNullQueryTypeScan(t *testing.T) {
- qType := repository.NullQuerytype{}
-
- stringType := "context_full"
- err := qType.Scan(stringType)
- require.NoError(t, err)
-}
-
-func TestNullQueryTypeValue(t *testing.T) {
- qType := repository.NullQuerytype{}
-
- stringType := "context_full"
- err := qType.Scan(stringType)
- require.NoError(t, err)
-
- val, err := qType.Value()
- require.NoError(t, err)
- assert.Equal(t, stringType, val)
-}
-
-func TestQueryTypeValid(t *testing.T) {
- assert.True(t, repository.QuerytypeContextFull.Valid())
-}
diff --git a/internal/database/repository/result.sql.go b/internal/database/repository/result.sql.go
deleted file mode 100644
index e3949114..00000000
--- a/internal/database/repository/result.sql.go
+++ /dev/null
@@ -1,308 +0,0 @@
-// Code generated by sqlc. DO NOT EDIT.
-// versions:
-// sqlc v1.27.0
-// source: result.sql
-
-package repository
-
-import (
- "context"
-
- "github.com/google/uuid"
-)
-
-const addResult = `-- name: AddResult :one
-INSERT INTO results (queryId, value, textEntryId, queryVersion) VALUES ($1, $2, $3, $4) returning id
-`
-
-type AddResultParams struct {
- Queryid uuid.UUID `db:"queryid"`
- Value string `db:"value"`
- Textentryid uuid.UUID `db:"textentryid"`
- Queryversion int32 `db:"queryversion"`
-}
-
-// AddResult
-//
-// INSERT INTO results (queryId, value, textEntryId, queryVersion) VALUES ($1, $2, $3, $4) returning id
-func (q *Queries) AddResult(ctx context.Context, arg *AddResultParams) (uuid.UUID, error) {
- row := q.db.QueryRow(ctx, addResult,
- arg.Queryid,
- arg.Value,
- arg.Textentryid,
- arg.Queryversion,
- )
- var id uuid.UUID
- err := row.Scan(&id)
- return id, err
-}
-
-const addResultDependency = `-- name: AddResultDependency :exec
-INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2)
-`
-
-type AddResultDependencyParams struct {
- Resultid uuid.UUID `db:"resultid"`
- Requiredresultid uuid.UUID `db:"requiredresultid"`
-}
-
-// AddResultDependency
-//
-// INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2)
-func (q *Queries) AddResultDependency(ctx context.Context, arg *AddResultDependencyParams) error {
- _, err := q.db.Exec(ctx, addResultDependency, arg.Resultid, arg.Requiredresultid)
- return err
-}
-
-const getResultValueWithVersion = `-- name: GetResultValueWithVersion :one
-WITH doc as (
- SELECT id, clientId
- FROM documents
- WHERE id = $3
-)
-SELECT r.id, r.value
- FROM doc as d
- JOIN currentTextEntries as cte on cte.documentId = d.id
- LEFT JOIN results as r
- on r.queryId = $1
- and r.queryVersion = $2
- and r.textEntryId = cte.id
-`
-
-type GetResultValueWithVersionParams struct {
- Queryid *uuid.UUID `db:"queryid"`
- Queryversion *int32 `db:"queryversion"`
- Documentid *uuid.UUID `db:"documentid"`
-}
-
-type GetResultValueWithVersionRow struct {
- ID *uuid.UUID `db:"id"`
- Value *string `db:"value"`
-}
-
-// GetResultValueWithVersion
-//
-// WITH doc as (
-// SELECT id, clientId
-// FROM documents
-// WHERE id = $3
-// )
-// SELECT r.id, r.value
-// FROM doc as d
-// JOIN currentTextEntries as cte on cte.documentId = d.id
-// LEFT JOIN results as r
-// on r.queryId = $1
-// and r.queryVersion = $2
-// and r.textEntryId = cte.id
-func (q *Queries) GetResultValueWithVersion(ctx context.Context, arg *GetResultValueWithVersionParams) (*GetResultValueWithVersionRow, error) {
- row := q.db.QueryRow(ctx, getResultValueWithVersion, arg.Queryid, arg.Queryversion, arg.Documentid)
- var i GetResultValueWithVersionRow
- err := row.Scan(&i.ID, &i.Value)
- return &i, err
-}
-
-const listQueryRequirementValues = `-- name: ListQueryRequirementValues :many
-WITH reqQueries as (
- SELECT av.queryId, av.activeVersion, q.queryType
- FROM requiredQueries as rq
- JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId
- JOIN queries as q on q.queryId = av.queryId
- WHERE rq.queryId = $1
- and isInVersion($2, rq.addedVersion, rq.removedVersion)
-),
-docs as (
- SELECT id, clientId
- FROM documents
- WHERE id = $3
-),
-codeVersions as (
- SELECT
- d.id as documentId,
- mcv.minCleanVersion,
- mtv.minTextVersion
- FROM docs as d
- JOIN currentCollectorMinTextVersions as mtv on mtv.clientId = d.clientId
- JOIN currentCollectorMinCleanVersions as mcv on mcv.clientId = d.clientId
-),
-latestVersions AS (
- SELECT
- r.id,
- rq.queryId,
- rq.queryType,
- r.queryVersion,
- r.textEntryId,
- r.value,
- ROW_NUMBER() OVER (
- PARTITION BY r.queryId
- ORDER BY r.id DESC
- ) as rowNumber
- FROM reqQueries as rq
- JOIN currentTextEntries as cte on cte.documentId = $3
- LEFT JOIN results as r ON rq.queryId = r.queryId
- and r.queryVersion = rq.activeVersion
- and cte.id = r.textEntryId
-)
-SELECT DISTINCT id, queryId, queryType, value
-FROM latestVersions
-WHERE rowNumber = 1
-`
-
-type ListQueryRequirementValuesParams struct {
- Queryid *uuid.UUID `db:"queryid"`
- Version *int32 `db:"version"`
- Documentid *uuid.UUID `db:"documentid"`
-}
-
-type ListQueryRequirementValuesRow struct {
- ID *uuid.UUID `db:"id"`
- Queryid uuid.UUID `db:"queryid"`
- Querytype Querytype `db:"querytype"`
- Value *string `db:"value"`
-}
-
-// ListQueryRequirementValues
-//
-// WITH reqQueries as (
-// SELECT av.queryId, av.activeVersion, q.queryType
-// FROM requiredQueries as rq
-// JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId
-// JOIN queries as q on q.queryId = av.queryId
-// WHERE rq.queryId = $1
-// and isInVersion($2, rq.addedVersion, rq.removedVersion)
-// ),
-// docs as (
-// SELECT id, clientId
-// FROM documents
-// WHERE id = $3
-// ),
-// codeVersions as (
-// SELECT
-// d.id as documentId,
-// mcv.minCleanVersion,
-// mtv.minTextVersion
-// FROM docs as d
-// JOIN currentCollectorMinTextVersions as mtv on mtv.clientId = d.clientId
-// JOIN currentCollectorMinCleanVersions as mcv on mcv.clientId = d.clientId
-// ),
-// latestVersions AS (
-// SELECT
-// r.id,
-// rq.queryId,
-// rq.queryType,
-// r.queryVersion,
-// r.textEntryId,
-// r.value,
-// ROW_NUMBER() OVER (
-// PARTITION BY r.queryId
-// ORDER BY r.id DESC
-// ) as rowNumber
-// FROM reqQueries as rq
-// JOIN currentTextEntries as cte on cte.documentId = $3
-// LEFT JOIN results as r ON rq.queryId = r.queryId
-// and r.queryVersion = rq.activeVersion
-// and cte.id = r.textEntryId
-// )
-// SELECT DISTINCT id, queryId, queryType, value
-// FROM latestVersions
-// WHERE rowNumber = 1
-func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQueryRequirementValuesParams) ([]*ListQueryRequirementValuesRow, error) {
- rows, err := q.db.Query(ctx, listQueryRequirementValues, arg.Queryid, arg.Version, arg.Documentid)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
- items := []*ListQueryRequirementValuesRow{}
- for rows.Next() {
- var i ListQueryRequirementValuesRow
- if err := rows.Scan(
- &i.ID,
- &i.Queryid,
- &i.Querytype,
- &i.Value,
- ); err != nil {
- return nil, err
- }
- items = append(items, &i)
- }
- if err := rows.Err(); err != nil {
- return nil, err
- }
- return items, nil
-}
-
-const listUnsyncedNoDepsQueriesByDocId = `-- name: ListUnsyncedNoDepsQueriesByDocId :many
-WITH docs as (
- SELECT id, clientId from documents where id = $1
-),
-dependency_tree as (
- select q.queryId, q.requiredIds, q.clientId, q.queryVersion
- from docs,
- LATERAL collectorQueryDependencyTreeByClient(docs.clientId) AS q
-),
-text_entries as (
- select id, documentId from currentTextEntries where documentId = $1
-),
-unsyncedQueries AS (
- SELECT DISTINCT dt.queryId, dt.requiredIds
- from docs as d
- JOIN dependency_tree as dt on d.clientId = dt.clientId
- JOIN text_entries as cte on cte.documentId = d.id
- LEFT JOIN results as r
- on r.queryId = dt.queryId
- and r.queryVersion = dt.queryVersion
- and cte.id = r.textEntryId
- where r.value is null
-)
-SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
- WHERE NOT EXISTS (
- SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds)
- )
-`
-
-// ListUnsyncedNoDepsQueriesByDocId
-//
-// WITH docs as (
-// SELECT id, clientId from documents where id = $1
-// ),
-// dependency_tree as (
-// select q.queryId, q.requiredIds, q.clientId, q.queryVersion
-// from docs,
-// LATERAL collectorQueryDependencyTreeByClient(docs.clientId) AS q
-// ),
-// text_entries as (
-// select id, documentId from currentTextEntries where documentId = $1
-// ),
-// unsyncedQueries AS (
-// SELECT DISTINCT dt.queryId, dt.requiredIds
-// from docs as d
-// JOIN dependency_tree as dt on d.clientId = dt.clientId
-// JOIN text_entries as cte on cte.documentId = d.id
-// LEFT JOIN results as r
-// on r.queryId = dt.queryId
-// and r.queryVersion = dt.queryVersion
-// and cte.id = r.textEntryId
-// where r.value is null
-// )
-// SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
-// WHERE NOT EXISTS (
-// SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds)
-// )
-func (q *Queries) ListUnsyncedNoDepsQueriesByDocId(ctx context.Context, dollar_1 *uuid.UUID) ([]*uuid.UUID, error) {
- rows, err := q.db.Query(ctx, listUnsyncedNoDepsQueriesByDocId, dollar_1)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
- items := []*uuid.UUID{}
- for rows.Next() {
- var queryid *uuid.UUID
- if err := rows.Scan(&queryid); err != nil {
- return nil, err
- }
- items = append(items, queryid)
- }
- if err := rows.Err(); err != nil {
- return nil, err
- }
- return items, nil
-}
diff --git a/internal/database/repository/result_test.go b/internal/database/repository/result_test.go
deleted file mode 100644
index 67a4b5ae..00000000
--- a/internal/database/repository/result_test.go
+++ /dev/null
@@ -1,533 +0,0 @@
-package repository_test
-
-import (
- "testing"
- "time"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/test"
-
- "github.com/google/uuid"
- "github.com/jackc/pgx/v5/pgtype"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestResults(t *testing.T) {
- t.Parallel()
- if testing.Short() {
- t.SkipNow()
- }
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
-
- qv := int32(1)
- res, err := queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{
- Queryid: &queryId,
- Queryversion: &qv,
- Documentid: &documentID,
- })
- require.NoError(t, err)
- assert.NotNil(t, res.Value)
- assert.Equal(t, "json_value", *res.Value)
-}
-
-func TestResultValues(t *testing.T) {
- t.Parallel()
- if testing.Short() {
- t.SkipNow()
- }
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
- require.NoError(t, err)
-
- clientId := "EXAMPLE"
- err = queries.CreateClient(ctx, &repository.CreateClientParams{
- Name: "example_client",
- Clientid: clientId,
- })
- require.NoError(t, err)
- documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
- Clientid: clientId,
- Hash: "example_hash",
- BatchID: nil,
- })
- require.NoError(t, err)
- documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
- Clientid: clientId,
- Hash: "example_hash_two",
- BatchID: nil,
- })
- require.NoError(t, err)
-
- contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
- require.NoError(t, err)
-
- _, err = queries.AddLatestQueryVersion(ctx, jsonQueryID)
- require.NoError(t, err)
- err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
- Queryid: jsonQueryID,
- Versionid: 1,
- })
- require.NoError(t, err)
- _, err = queries.AddLatestQueryVersion(ctx, contextQueryID)
- require.NoError(t, err)
- _, err = queries.AddLatestQueryVersion(ctx, contextQueryID)
- require.NoError(t, err)
- err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
- Queryid: contextQueryID,
- Versionid: 2,
- })
- require.NoError(t, err)
-
- jsonVersion := int32(1)
- err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
- Queryid: jsonQueryID,
- Requiredqueryid: contextQueryID,
- Addedversion: jsonVersion,
- })
- require.NoError(t, err)
-
- contextQuery, err := queries.GetQuery(ctx, contextQueryID)
- require.NoError(t, err)
-
- qResults, err := queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
- Queryid: &jsonQueryID,
- Documentid: &documentID,
- Version: &jsonVersion,
- })
- require.NoError(t, err)
- assert.Len(t, qResults, 0)
- qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
- Queryid: &jsonQueryID,
- Documentid: &documentTwoID,
- Version: &jsonVersion,
- })
- require.NoError(t, err)
- assert.Len(t, qResults, 0)
-
- version, err := queries.AddLatestCollectorVersion(ctx, clientId)
- require.NoError(t, err)
- err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
- Versionid: version,
- Clientid: clientId,
- })
- require.NoError(t, err)
-
- bucket := "example_bucket"
- key := "example_key"
- hash := "hash"
- cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
- Documentid: documentID,
- Bucket: &bucket,
- Key: &key,
- Hash: &hash,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanid,
- Version: 1,
- })
- require.NoError(t, err)
- textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
- Cleanid: cleanid,
- Bucket: "hi",
- Key: "hello",
- Hash: "example",
- Part: 0,
- Createdat: pgtype.Timestamp{
- Time: time.Now().UTC(),
- Valid: true,
- },
- })
- require.NoError(t, err)
- err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
- Version: 1,
- Textid: textId,
- })
- require.NoError(t, err)
-
- result := repository.AddResultParams{
- Queryid: contextQueryID,
- Value: "context_value_1",
- Textentryid: textId,
- Queryversion: contextQuery.Activeversion,
- }
-
- _, err = queries.AddResult(ctx, &result)
- require.NoError(t, err)
-
- qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
- Queryid: &jsonQueryID,
- Documentid: &documentID,
- Version: &jsonVersion,
- })
- require.NoError(t, err)
- assert.Len(t, qResults, 1)
- assert.Equal(t, contextQueryID, qResults[0].Queryid)
- assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Querytype)
- assert.Equal(t, "context_value_1", *qResults[0].Value)
- assert.NotEqual(t, uuid.UUID{}, qResults[0].ID)
- qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
- Queryid: &jsonQueryID,
- Documentid: &documentTwoID,
- Version: &jsonVersion,
- })
- require.NoError(t, err)
- assert.Len(t, qResults, 0)
-
- _, err = queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: contextQueryID,
- Value: "context_value_2",
- Textentryid: textId,
- Queryversion: contextQuery.Activeversion - 1,
- })
- require.NoError(t, err)
-
- qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
- Queryid: &jsonQueryID,
- Documentid: &documentID,
- Version: &jsonVersion,
- })
- require.NoError(t, err)
- assert.Len(t, qResults, 1)
- assert.Equal(t, contextQueryID, qResults[0].Queryid)
- assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Querytype)
- assert.Equal(t, "context_value_1", *qResults[0].Value)
- assert.NotEqual(t, uuid.UUID{}, qResults[0].ID)
-
- _, err = queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: contextQueryID,
- Value: "context_value_3",
- Textentryid: textId,
- Queryversion: contextQuery.Activeversion,
- })
- require.NoError(t, err)
-
- qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
- Queryid: &jsonQueryID,
- Documentid: &documentID,
- Version: &jsonVersion,
- })
- require.NoError(t, err)
- assert.Len(t, qResults, 1)
- assert.Equal(t, contextQueryID, qResults[0].Queryid)
- assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Querytype)
- assert.Equal(t, "context_value_3", *qResults[0].Value)
- assert.NotEqual(t, uuid.UUID{}, qResults[0].ID)
-
- _, err = queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: jsonQueryID,
- Value: "json_value_1",
- Textentryid: textId,
- Queryversion: jsonVersion,
- })
- require.NoError(t, err)
-
- qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
- Queryid: &jsonQueryID,
- Documentid: &documentID,
- Version: &jsonVersion,
- })
- require.NoError(t, err)
- assert.Len(t, qResults, 1)
- assert.Equal(t, contextQueryID, qResults[0].Queryid)
- assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Querytype)
- assert.Equal(t, "context_value_3", *qResults[0].Value)
- assert.NotEqual(t, uuid.UUID{}, qResults[0].ID)
-}
-
-func TestUnsyncedNoDepsQueries(t *testing.T) {
- if testing.Short() {
- t.SkipNow()
- }
- t.Run("single document", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- clientId := createClientWithCollector(t, queries)
- contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
-
- err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
- Clientid: clientId,
- Name: "example_name",
- Queryid: jsonQueryID,
- Addedversion: 1,
- })
- require.NoError(t, err)
-
- documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
- Clientid: clientId,
- Hash: "example_hash",
- })
- require.NoError(t, err)
-
- qs, err := queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID)
- require.NoError(t, err)
- assert.Len(t, qs, 0)
-
- bucket := "example_bucket"
- key := "example_key"
- hash := "hahs"
- cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
- Documentid: documentID,
- Bucket: &bucket,
- Key: &key,
- Hash: &hash,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanid,
- Version: 1,
- })
- require.NoError(t, err)
-
- qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID)
- require.NoError(t, err)
- assert.Len(t, qs, 0)
-
- textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
- Cleanid: cleanid,
- Bucket: "hi",
- Key: "hello",
- Part: 0,
- Createdat: pgtype.Timestamp{
- Time: time.Now().UTC(),
- Valid: true,
- },
- })
- require.NoError(t, err)
- err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
- Version: 1,
- Textid: textId,
- })
- require.NoError(t, err)
-
- qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID)
- require.NoError(t, err)
- assert.Len(t, qs, 1)
- assert.ElementsMatch(t, []*uuid.UUID{&contextQueryID}, qs)
-
- _, err = queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: contextQueryID,
- Value: "context_value",
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID)
- require.NoError(t, err)
- assert.Len(t, qs, 1)
- assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, qs)
-
- _, err = queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: jsonQueryID,
- Value: "context_value",
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID)
- require.NoError(t, err)
- assert.Len(t, qs, 0)
-
- _, err = queries.AddLatestQueryVersion(ctx, jsonQueryID)
- require.NoError(t, err)
- err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
- Queryid: jsonQueryID,
- Versionid: 2,
- })
- require.NoError(t, err)
-
- qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID)
- require.NoError(t, err)
- assert.Len(t, qs, 1)
- assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, qs)
- })
-
- t.Run("two files", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- clientId := createClientWithCollector(t, queries)
- contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
- documentID, _, _ := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID)
-
- documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
- Clientid: clientId,
- Hash: "example_hash_two",
- })
- require.NoError(t, err)
-
- bucket := "example_bucket"
- key := "example_key"
- hash := "hahs"
-
- cleantwoid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
- Documentid: documentTwoID,
- Bucket: &bucket,
- Key: &key,
- Hash: &hash,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
- Cleanid: cleantwoid,
- Version: 1,
- })
- require.NoError(t, err)
-
- qs, err := queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID)
- require.NoError(t, err)
- assert.Len(t, qs, 0)
- qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID)
- require.NoError(t, err)
- assert.Len(t, qs, 0)
-
- textTwoId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
- Cleanid: cleantwoid,
- Bucket: "hi",
- Key: "hello",
- Part: 0,
- Createdat: pgtype.Timestamp{
- Time: time.Now().UTC(),
- Valid: true,
- },
- })
- require.NoError(t, err)
- err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
- Version: 1,
- Textid: textTwoId,
- })
- require.NoError(t, err)
-
- _, err = queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: contextQueryID,
- Value: "context_value",
- Textentryid: textTwoId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID)
- require.NoError(t, err)
- assert.Len(t, qs, 0)
- qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID)
- require.NoError(t, err)
- assert.Len(t, qs, 1)
- assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, qs)
-
- _, err = queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: jsonQueryID,
- Value: "context_value",
- Textentryid: textTwoId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID)
- require.NoError(t, err)
- assert.Len(t, qs, 0)
- qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID)
- require.NoError(t, err)
- assert.Len(t, qs, 0)
- })
-
- t.Run("update query", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- clientId := createClientWithCollector(t, queries)
- contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
- documentID, _, _ := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID)
-
- _, err := queries.AddLatestQueryVersion(ctx, jsonQueryID)
- require.NoError(t, err)
- err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
- Queryid: jsonQueryID,
- Versionid: 2,
- })
- require.NoError(t, err)
-
- qs, err := queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID)
- require.NoError(t, err)
- assert.Len(t, qs, 1)
- assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, qs)
- })
-}
-
-func BenchmarkListUnsynced(b *testing.B) {
- ctx := b.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(b, cfg)
-
- queries := cfg.GetDBQueries()
-
- clientId := createClientWithCollector(b, queries)
- _, jsonQueryID := createDependentQueries(b, ctx, queries)
-
- err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
- Clientid: clientId,
- Name: "example_name",
- Queryid: jsonQueryID,
- Addedversion: 1,
- })
- require.NoError(b, err)
-
- documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
- Clientid: clientId,
- Hash: "example_hash",
- BatchID: nil,
- })
- require.NoError(b, err)
-
- b.ResetTimer()
-
- for b.Loop() {
- _, _ = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID)
- }
-}
diff --git a/internal/database/repository/sync_test.go b/internal/database/repository/sync_test.go
deleted file mode 100644
index 9cbeb9a1..00000000
--- a/internal/database/repository/sync_test.go
+++ /dev/null
@@ -1,1144 +0,0 @@
-package repository_test
-
-import (
- "context"
- "testing"
- "time"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/test"
-
- "github.com/google/uuid"
- "github.com/jackc/pgx/v5/pgtype"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestListClientDocumentIDs(t *testing.T) {
- t.Parallel()
- if testing.Short() {
- t.SkipNow()
- }
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- id := "EXAMPLE"
- err := queries.CreateClient(ctx, &repository.CreateClientParams{
- Name: "example_client",
- Clientid: id,
- })
- require.NoError(t, err)
-
- ids, err := queries.ListDocumentIDsBatch(ctx, &repository.ListDocumentIDsBatchParams{
- Clientid: id,
- Batchsize: 1,
- Pageoffset: 0,
- })
- require.NoError(t, err)
- assert.Len(t, ids, 0)
-
- docOne, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
- Clientid: id,
- Hash: "example_hash",
- BatchID: nil,
- })
- require.NoError(t, err)
-
- ids, err = queries.ListDocumentIDsBatch(ctx, &repository.ListDocumentIDsBatchParams{
- Clientid: id,
- Batchsize: 1,
- Pageoffset: 0,
- })
- require.NoError(t, err)
- assert.Len(t, ids, 1)
- total := int64(1)
- assert.ElementsMatch(t, []*repository.ListDocumentIDsBatchRow{
- {
- ID: &docOne,
- Totalcount: &total,
- },
- }, ids)
-
- docTwo, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
- Clientid: id,
- Hash: "example_hash_two",
- BatchID: nil,
- })
- require.NoError(t, err)
-
- ids, err = queries.ListDocumentIDsBatch(ctx, &repository.ListDocumentIDsBatchParams{
- Clientid: id,
- Batchsize: 1,
- Pageoffset: 0,
- })
- require.NoError(t, err)
- assert.Len(t, ids, 1)
- total = int64(2)
- assert.ElementsMatch(t, []*repository.ListDocumentIDsBatchRow{
- {
- ID: &docOne,
- Totalcount: &total,
- },
- }, ids)
-
- ids, err = queries.ListDocumentIDsBatch(ctx, &repository.ListDocumentIDsBatchParams{
- Clientid: id,
- Batchsize: 1,
- Pageoffset: 1,
- })
- require.NoError(t, err)
- assert.Len(t, ids, 1)
- assert.ElementsMatch(t, []*repository.ListDocumentIDsBatchRow{
- {
- ID: &docTwo,
- Totalcount: &total,
- },
- }, ids)
-
- ids, err = queries.ListDocumentIDsBatch(ctx, &repository.ListDocumentIDsBatchParams{
- Clientid: id,
- Batchsize: 2,
- Pageoffset: 0,
- })
- require.NoError(t, err)
- assert.Len(t, ids, 2)
- assert.ElementsMatch(t, []*repository.ListDocumentIDsBatchRow{
- {
- ID: &docOne,
- Totalcount: &total,
- },
- {
- ID: &docTwo,
- Totalcount: &total,
- },
- }, ids)
-}
-
-type docSyncStateParams struct {
- clientId string
- documentId uuid.UUID
- hash string
- fields string
- isSynced bool
-}
-
-func getDocumentSyncState(t testing.TB, ctx context.Context, queries *repository.Queries, params docSyncStateParams) {
- t.Helper()
- isSynced, err := queries.IsClientSynced(ctx, ¶ms.clientId)
- require.NoError(t, err)
- assert.Equal(t, params.isSynced, isSynced)
-
- doc, err := queries.GetDocumentExternal(ctx, params.documentId)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{
- ID: params.documentId,
- Clientid: params.clientId,
- Hash: params.hash,
- Fields: []byte(params.fields),
- }, doc)
-}
-
-func createQuery(t testing.TB, queries *repository.Queries, queryType repository.Querytype) uuid.UUID {
- id, err := queries.CreateQuery(t.Context(), queryType)
- require.NoError(t, err)
-
- version, err := queries.AddLatestQueryVersion(t.Context(), id)
- require.NoError(t, err)
-
- err = queries.AddActiveQueryVersion(t.Context(), &repository.AddActiveQueryVersionParams{
- Queryid: id,
- Versionid: version,
- })
- require.NoError(t, err)
-
- return id
-}
-
-func createDependentQueries(t testing.TB, ctx context.Context, queries *repository.Queries) (uuid.UUID, uuid.UUID) {
- contextQueryID := createQuery(t, queries, repository.QuerytypeContextFull)
- jsonQueryID := createQuery(t, queries, repository.QuerytypeJsonExtractor)
-
- err := queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
- Queryid: jsonQueryID,
- Requiredqueryid: contextQueryID,
- Addedversion: 1,
- })
- require.NoError(t, err)
-
- return contextQueryID, jsonQueryID
-}
-
-func createClientWithCollector(t testing.TB, queries *repository.Queries) string {
- clientId := "EXAMPLE"
- err := queries.CreateClient(t.Context(), &repository.CreateClientParams{
- Name: "example_client",
- Clientid: clientId,
- })
- require.NoError(t, err)
-
- collectorVersion, err := queries.AddLatestCollectorVersion(t.Context(), clientId)
- require.NoError(t, err)
- err = queries.SetActiveCollectorVersion(t.Context(), &repository.SetActiveCollectorVersionParams{
- Versionid: collectorVersion,
- Clientid: clientId,
- })
- require.NoError(t, err)
-
- return clientId
-}
-
-func TestClientSync(t *testing.T) {
- if testing.Short() {
- t.SkipNow()
- }
-
- t.Run("no collector", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- clientId := "EXAMPLE"
- err := queries.CreateClient(ctx, &repository.CreateClientParams{
- Name: "example_client",
- Clientid: clientId,
- })
- require.NoError(t, err)
-
- issynced, err := queries.IsClientSynced(ctx, &clientId)
- require.NoError(t, err)
- assert.True(t, issynced)
- })
-
- t.Run("no documents", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- _, jsonQueryID := createDependentQueries(t, ctx, queries)
- clientId := createClientWithCollector(t, queries)
-
- isSynced, err := queries.IsClientSynced(ctx, &clientId)
- require.NoError(t, err)
- assert.True(t, isSynced)
-
- err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
- Clientid: clientId,
- Queryid: jsonQueryID,
- Addedversion: 1,
- Name: "first_key",
- })
- require.NoError(t, err)
-
- isSynced, err = queries.IsClientSynced(ctx, &clientId)
- require.NoError(t, err)
- assert.True(t, isSynced)
- })
-
- t.Run("create document", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
-
- err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
- Clientid: clientId,
- Queryid: queryId,
- Addedversion: 1,
- Name: "first_key",
- })
- require.NoError(t, err)
-
- documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
- Clientid: clientId,
- Hash: "example_noclean",
- BatchID: nil,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_noclean",
- isSynced: false,
- fields: `{"first_key": null}`,
- })
- })
-
- t.Run("document fail clean", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID := createDocumentWithCollector(t, queries, clientId, queryId)
-
- cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
- Documentid: documentID,
- Fail: repository.NullCleanfailtype{
- Valid: true,
- Cleanfailtype: repository.CleanfailtypeInvalidMimetype,
- },
- })
- require.NoError(t, err)
- err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanid,
- Version: 1,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: true,
- fields: `{"first_key": null}`,
- })
- })
-
- t.Run("clean document", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID := createDocumentWithCollector(t, queries, clientId, queryId)
-
- bucket := "buck"
- key := "example_key"
- hash := "hahssh"
- cleanId, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
- Documentid: documentID,
- Bucket: &bucket,
- Key: &key,
- Hash: &hash,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanId,
- Version: 1,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: false,
- fields: `{"first_key": null}`,
- })
- })
-
- t.Run("extracted text", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID, cleanId := createCleanDocumentWithCollector(t, queries, clientId, queryId)
-
- textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
- Cleanid: cleanId,
- Part: 0,
- Createdat: pgtype.Timestamp{
- Time: time.Now().UTC(),
- Valid: true,
- },
- Bucket: "hi",
- Key: "hello",
- Hash: "example",
- })
- require.NoError(t, err)
- err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
- Version: 1,
- Textid: textId,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: false,
- fields: `{"first_key": null}`,
- })
- })
-
- t.Run("single query", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID, _, textId := createDocumentWithCollectorAndText(t, queries, clientId, queryId)
-
- _, err := queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: queryId,
- Value: "json_value",
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: true,
- fields: `{"first_key": "json_value"}`,
- })
- })
-
- t.Run("change super query", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
- clientId := createClientWithCollector(t, queries)
- documentID, _, textId := createDocumentWithCollectorAndText(t, queries, clientId, jsonQueryID)
-
- _, err := queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: contextQueryID,
- Value: "example_context",
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: false,
- fields: `{"first_key": null}`,
- })
-
- })
- t.Run("upstream query update result", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
- clientId := createClientWithCollector(t, queries)
- documentID, _, textId := createDocumentWithCollectorAndText(t, queries, clientId, jsonQueryID)
-
- contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: contextQueryID,
- Value: "example_context",
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: jsonQueryID,
- Value: "json_value",
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
- Resultid: jsonResultId,
- Requiredresultid: contextResultId,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: true,
- fields: `{"first_key": "json_value"}`,
- })
- })
-
- t.Run("update upstream query", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
- clientId := createClientWithCollector(t, queries)
- documentID, _, textId := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID)
-
- contextLatestVersion, err := queries.AddLatestQueryVersion(ctx, contextQueryID)
- require.NoError(t, err)
- err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
- Queryid: contextQueryID,
- Versionid: contextLatestVersion,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: false,
- fields: `{"first_key": null}`,
- })
-
- contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: contextQueryID,
- Value: "context_version_2",
- Textentryid: textId,
- Queryversion: contextLatestVersion,
- })
- require.NoError(t, err)
-
- jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: jsonQueryID,
- Value: "updated_context_value",
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
- Resultid: jsonResultId,
- Requiredresultid: contextResultId,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: true,
- fields: `{"first_key": "updated_context_value"}`,
- })
- })
-
- t.Run("update text entry", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID, cleanId, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
-
- textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
- Bucket: "hi",
- Key: "hello",
- Part: 0,
- Createdat: pgtype.Timestamp{
- Time: time.Now().UTC(),
- Valid: true,
- },
- Cleanid: cleanId,
- })
- require.NoError(t, err)
- err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
- Version: 1,
- Textid: textId,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: false,
- fields: `{"first_key": null}`,
- })
- })
- t.Run("update text entry result", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID, cleanId, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
-
- textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
- Bucket: "hi",
- Key: "hello",
- Part: 0,
- Createdat: pgtype.Timestamp{
- Time: time.Now().UTC(),
- Valid: true,
- },
- Cleanid: cleanId,
- })
- require.NoError(t, err)
- err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
- Version: 1,
- Textid: textId,
- })
- require.NoError(t, err)
-
- _, err = queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: queryId,
- Value: "updated_text_json",
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: true,
- fields: `{"first_key": "updated_text_json"}`,
- })
- })
- t.Run("update clean entry", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
-
- bucket := "cu"
- key := "kk"
- hash := "hash"
- cleanthreeid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
- Documentid: documentID,
- Bucket: &bucket,
- Key: &key,
- Hash: &hash,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanthreeid,
- Version: 1,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: false,
- fields: `{"first_key": null}`,
- })
- })
- t.Run("update clean entry with result", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
-
- bucket := "cu"
- key := "kk"
- hash := "hash"
- cleanthreeid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
- Documentid: documentID,
- Bucket: &bucket,
- Key: &key,
- Hash: &hash,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanthreeid,
- Version: 1,
- })
- require.NoError(t, err)
-
- textThreeId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
- Bucket: "hi",
- Key: "hello",
- Part: 0,
- Createdat: pgtype.Timestamp{
- Time: time.Now().UTC(),
- Valid: true,
- },
- Cleanid: cleanthreeid,
- })
- require.NoError(t, err)
- err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
- Version: 1,
- Textid: textThreeId,
- })
- require.NoError(t, err)
-
- _, err = queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: queryId,
- Value: "update_clean_json",
- Textentryid: textThreeId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: true,
- fields: `{"first_key": "update_clean_json"}`,
- })
- })
-
- t.Run("add new collector version", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
-
- _, err := queries.AddLatestCollectorVersion(ctx, clientId)
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: true,
- fields: `{"first_key": "json_value"}`,
- })
- })
-
- t.Run("add new active collector version", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
-
- latestCollectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId)
- require.NoError(t, err)
-
- err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
- Versionid: latestCollectorVersion,
- Clientid: clientId,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: true,
- fields: `{"first_key": "json_value"}`,
- })
- })
-
- t.Run("remove query from new collector version", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
-
- latestCollectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId)
- require.NoError(t, err)
-
- err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
- Clientid: clientId,
- Queryid: queryId,
- Removedversion: &latestCollectorVersion,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: true,
- fields: `{"first_key": "json_value"}`,
- })
- })
- t.Run("remove query and update collector version", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
-
- latestCollectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId)
- require.NoError(t, err)
-
- err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
- Clientid: clientId,
- Queryid: queryId,
- Removedversion: &latestCollectorVersion,
- })
- require.NoError(t, err)
-
- err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
- Versionid: latestCollectorVersion,
- Clientid: clientId,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: true,
- fields: `{}`,
- })
- })
-
- t.Run("add existing query", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
- clientId := createClientWithCollector(t, queries)
- documentID, _, _ := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID)
-
- err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
- Clientid: clientId,
- Queryid: contextQueryID,
- Addedversion: 1,
- Name: "example_key",
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: true,
- fields: `{"first_key": "json_value", "example_key": "example_context"}`,
- })
- })
-
- t.Run("add super query", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
-
- superQueryID := createQuery(t, queries, repository.QuerytypeJsonExtractor)
- err := queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
- Queryid: queryId,
- Requiredqueryid: superQueryID,
- Addedversion: 1,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: false,
- fields: `{"first_key": null}`,
- })
- })
- t.Run("add super query with result", func(t *testing.T) {
- t.Parallel()
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- queries := cfg.GetDBQueries()
-
- queryId := createQuery(t, queries, repository.QuerytypeContextFull)
- clientId := createClientWithCollector(t, queries)
- documentID, _, textId := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
-
- superQueryID := createQuery(t, queries, repository.QuerytypeJsonExtractor)
- err := queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
- Queryid: queryId,
- Requiredqueryid: superQueryID,
- Addedversion: 1,
- })
- require.NoError(t, err)
-
- superResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: superQueryID,
- Value: "super_value",
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- resultId, err := queries.AddResult(ctx, &repository.AddResultParams{
- Queryid: queryId,
- Value: "json_with_super",
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
- Resultid: resultId,
- Requiredresultid: superResultId,
- })
- require.NoError(t, err)
-
- getDocumentSyncState(t, ctx, queries, docSyncStateParams{
- clientId: clientId,
- documentId: documentID,
- hash: "example_hash",
- isSynced: true,
- fields: `{"first_key": "json_with_super"}`,
- })
- })
-}
-
-func createDocumentWithCollector(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) uuid.UUID {
- err := queries.AddCollectorQuery(t.Context(), &repository.AddCollectorQueryParams{
- Clientid: clientId,
- Queryid: jsonQueryID,
- Addedversion: 1,
- Name: "first_key",
- })
- require.NoError(t, err)
-
- documentID, err := queries.CreateDocument(t.Context(), &repository.CreateDocumentParams{
- Clientid: clientId,
- Hash: "example_hash",
- BatchID: nil,
- })
- require.NoError(t, err)
-
- return documentID
-}
-
-func createCleanDocumentWithCollector(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID) {
- documentID := createDocumentWithCollector(t, queries, clientId, jsonQueryID)
-
- bucket := "example_bucket"
- key := "example_key"
- hash := "hahssh"
- cleanId, err := queries.AddDocumentClean(t.Context(), &repository.AddDocumentCleanParams{
- Documentid: documentID,
- Bucket: &bucket,
- Key: &key,
- Hash: &hash,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = queries.AddDocumentCleanEntry(t.Context(), &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanId,
- Version: 1,
- })
- require.NoError(t, err)
-
- return documentID, cleanId
-}
-
-func createDocumentWithCollectorAndText(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) {
- documentID, cleanId := createCleanDocumentWithCollector(t, queries, clientId, jsonQueryID)
-
- textId, err := queries.AddDocumentText(t.Context(), &repository.AddDocumentTextParams{
- Cleanid: cleanId,
- Part: 0,
- Createdat: pgtype.Timestamp{
- Time: time.Now().UTC(),
- Valid: true,
- },
- Bucket: "hi",
- Key: "hello",
- Hash: "example",
- })
- require.NoError(t, err)
- err = queries.AddDocumentTextEntry(t.Context(), &repository.AddDocumentTextEntryParams{
- Version: 1,
- Textid: textId,
- })
- require.NoError(t, err)
-
- return documentID, cleanId, textId
-}
-
-func createDocumentWithCollectorAndResult(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) {
- documentID, cleanId, textId := createDocumentWithCollectorAndText(t, queries, clientId, jsonQueryID)
-
- _, err := queries.AddResult(t.Context(), &repository.AddResultParams{
- Queryid: jsonQueryID,
- Value: "json_value",
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- return documentID, cleanId, textId
-}
-
-func createDocumentWithCollectorAndResults(t testing.TB, queries *repository.Queries, clientId string, contextQueryID, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) {
- documentID, cleanId, textId := createDocumentWithCollectorAndText(t, queries, clientId, jsonQueryID)
-
- contextResultId, err := queries.AddResult(t.Context(), &repository.AddResultParams{
- Queryid: contextQueryID,
- Value: "example_context",
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- jsonResultId, err := queries.AddResult(t.Context(), &repository.AddResultParams{
- Queryid: jsonQueryID,
- Value: "json_value",
- Textentryid: textId,
- Queryversion: 1,
- })
- require.NoError(t, err)
-
- err = queries.AddResultDependency(t.Context(), &repository.AddResultDependencyParams{
- Resultid: jsonResultId,
- Requiredresultid: contextResultId,
- })
- require.NoError(t, err)
-
- return documentID, cleanId, textId
-}
-
-func BenchmarkIsClientSynced(b *testing.B) {
- ctx := b.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(b, cfg)
-
- queries := cfg.GetDBQueries()
-
- contextQueryID, jsonQueryID := createDependentQueries(b, ctx, queries)
- clientId := createClientWithCollector(b, queries)
- _, _, _ = createDocumentWithCollectorAndResults(b, queries, clientId, contextQueryID, jsonQueryID)
-
- b.ResetTimer()
-
- for b.Loop() {
- _, _ = queries.IsClientSynced(ctx, &clientId)
- }
-}
-
-func BenchmarkGetDocumentExternal(b *testing.B) {
- ctx := b.Context()
-
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(b, cfg)
-
- queries := cfg.GetDBQueries()
-
- contextQueryID, jsonQueryID := createDependentQueries(b, ctx, queries)
- clientId := createClientWithCollector(b, queries)
- documentID, _, _ := createDocumentWithCollectorAndResults(b, queries, clientId, contextQueryID, jsonQueryID)
-
- b.ResetTimer()
-
- for b.Loop() {
- _, _ = queries.GetDocumentExternal(ctx, documentID)
- }
-}
diff --git a/internal/document/get.go b/internal/document/get.go
index 48bda745..530654e1 100644
--- a/internal/document/get.go
+++ b/internal/document/get.go
@@ -2,7 +2,6 @@ package document
import (
"context"
- "encoding/json"
"github.com/google/uuid"
)
@@ -21,24 +20,18 @@ func (s *Service) GetSummary(ctx context.Context, id uuid.UUID) (*DocumentSummar
}, nil
}
-// GetExternal returns document info with computed fields for external API responses.
+// GetExternal returns document info for external API responses.
+// Note: Query functionality (Fields) has been removed. See remove_query_plan.md for details.
func (s *Service) GetExternal(ctx context.Context, id uuid.UUID) (*DocumentExternal, error) {
doc, err := s.cfg.GetDBQueries().GetDocumentExternal(ctx, id)
if err != nil {
return nil, err
}
- var fields map[string]interface{}
- err = json.Unmarshal(doc.Fields, &fields)
- if err != nil {
- return nil, err
- }
-
return &DocumentExternal{
Id: doc.ID,
ClientID: doc.Clientid,
Hash: doc.Hash,
- Fields: fields,
}, nil
}
@@ -49,21 +42,16 @@ func (s *Service) GetExternal(ctx context.Context, id uuid.UUID) (*DocumentExter
// - includeTextRecord: if true, fetches and includes the full field extraction response
//
// Returns DocumentEnriched with all metadata, or error if document not found or other failure.
+// Note: Query functionality (Fields) has been removed. See remove_query_plan.md for details.
func (s *Service) GetEnriched(ctx context.Context, id uuid.UUID, includeTextRecord bool) (*DocumentEnriched, error) {
queries := s.cfg.GetDBQueries()
- // 1. Get basic document info + computed fields
+ // 1. Get basic document info
docExternal, err := queries.GetDocumentExternal(ctx, id)
if err != nil {
return nil, err
}
- var fields map[string]interface{}
- err = json.Unmarshal(docExternal.Fields, &fields)
- if err != nil {
- return nil, err
- }
-
// 2. Get extended document metadata (folder, filename, originalPath)
docEnriched, err := queries.GetDocumentEnriched(ctx, id)
if err != nil {
@@ -97,7 +85,6 @@ func (s *Service) GetEnriched(ctx context.Context, id uuid.UUID, includeTextReco
ID: docExternal.ID,
ClientID: docExternal.Clientid,
Hash: docExternal.Hash,
- Fields: fields,
HasTextRecord: hasTextRecord,
FolderID: docEnriched.Folderid,
Filename: docEnriched.Filename,
diff --git a/internal/document/get_test.go b/internal/document/get_test.go
index cf8f63a9..d2befd49 100644
--- a/internal/document/get_test.go
+++ b/internal/document/get_test.go
@@ -59,19 +59,17 @@ func TestGetExternal(t *testing.T) {
svc := document.New(cfg)
+ // Note: Query functionality (Fields) has been removed. See remove_query_plan.md for details.
doc := document.DocumentExternal{
Id: uuid.New(),
ClientID: "externalID",
Hash: "example",
- Fields: map[string]interface{}{
- "json": "hello",
- },
}
pool.ExpectQuery("name: GetDocumentExternal :one").WithArgs(doc.Id).
WillReturnRows(
- pgxmock.NewRows([]string{"id", "clientId", "hash", "fields"}).
- AddRow(doc.Id, "externalID", "example", []byte(`{"json": "hello"}`)),
+ pgxmock.NewRows([]string{"id", "clientId", "hash"}).
+ AddRow(doc.Id, "externalID", "example"),
)
adoc, err := svc.GetExternal(ctx, doc.Id)
diff --git a/internal/document/service.go b/internal/document/service.go
index 56b481e5..3b330dd0 100644
--- a/internal/document/service.go
+++ b/internal/document/service.go
@@ -15,21 +15,21 @@ type DocumentSummary struct {
Hash string
}
-// DocumentExternal contains document info with computed fields for external API responses.
+// DocumentExternal contains document info for external API responses.
+// Note: Query functionality (Fields) has been removed. See remove_query_plan.md for details.
type DocumentExternal struct {
Id uuid.UUID
ClientID string
Hash string
- Fields map[string]interface{}
}
// DocumentEnriched contains full document details including metadata, labels, and optionally text record.
// Used by GET /document/{id} endpoint.
+// Note: Query functionality (Fields) has been removed. See remove_query_plan.md for details.
type DocumentEnriched struct {
ID uuid.UUID
ClientID string
Hash string
- Fields map[string]interface{}
HasTextRecord bool
FolderID *uuid.UUID
Filename *string
diff --git a/internal/document/text/extract.go b/internal/document/text/extract.go
index bb0dd59f..61550ddd 100644
--- a/internal/document/text/extract.go
+++ b/internal/document/text/extract.go
@@ -4,12 +4,12 @@ import (
"context"
"log/slog"
- querysyncrunner "queryorchestration/api/querySyncRunner"
- "queryorchestration/internal/serviceconfig/queue"
-
"github.com/google/uuid"
)
+// Extract extracts text from a document using AWS Textract.
+// This is the terminal step in the document processing pipeline.
+// Query processing has been removed - pipeline ends after text extraction.
func (s *Service) Extract(ctx context.Context, documentId uuid.UUID) error {
slog.Debug("extracting document text", "id", documentId.String())
@@ -25,19 +25,6 @@ func (s *Service) Extract(ctx context.Context, documentId uuid.UUID) error {
}
}
- return s.informExtraction(ctx, documentId)
-}
-
-func (s *Service) informExtraction(ctx context.Context, id uuid.UUID) error {
- err := s.cfg.SendToQueue(ctx, &queue.SendParams{
- QueueURL: s.cfg.GetQuerySyncURL(),
- Body: querysyncrunner.Body{
- DocumentID: id,
- },
- })
- if err != nil {
- return err
- }
-
+ // Pipeline terminates here - query processing has been removed
return nil
}
diff --git a/internal/document/text/extract_test.go b/internal/document/text/extract_test.go
index 06fa2133..1ec464cd 100644
--- a/internal/document/text/extract_test.go
+++ b/internal/document/text/extract_test.go
@@ -1,7 +1,6 @@
package documenttext
import (
- "fmt"
"io"
"strings"
"testing"
@@ -11,14 +10,11 @@ import (
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/aws"
"queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/textract"
objectstoremock "queryorchestration/mocks/objectstore"
- queuemock "queryorchestration/mocks/queue"
textractmock "queryorchestration/mocks/textract"
"github.com/aws/aws-sdk-go-v2/service/s3"
- "github.com/aws/aws-sdk-go-v2/service/sqs"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/mock"
@@ -28,7 +24,6 @@ import (
type DocTextConfig struct {
aws.AWSConfig
serviceconfig.BaseConfig
- querysync.QuerySyncConfig
textract.TextractConfig
objectstore.ObjectStoreConfig
}
@@ -45,9 +40,6 @@ func TestCreate(t *testing.T) {
cfg.StoreClient = mockStore
mockTextract := textractmock.NewMockTextractClient(t)
cfg.TextractClient = mockTextract
- mockQueue := queuemock.NewMockSQSClient(t)
- cfg.QueueClient = mockQueue
- cfg.QuerySyncURL = "ex"
svc := Service{
cfg: cfg,
@@ -107,16 +99,6 @@ func TestCreate(t *testing.T) {
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
- mockQueue.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", docId.String())
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
err = svc.Extract(ctx, docId)
require.NoError(t, err)
})
@@ -126,43 +108,7 @@ func TestCreate(t *testing.T) {
AddRow(true),
)
- mockQueue.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", docId.String())
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
err = svc.Extract(ctx, docId)
require.NoError(t, err)
})
}
-
-func TestInformClean(t *testing.T) {
- ctx := t.Context()
- mockSQS := queuemock.NewMockSQSClient(t)
-
- cfg := &DocTextConfig{}
- cfg.QueueClient = mockSQS
- cfg.QuerySyncURL = "/i/am/here"
- svc := Service{
- cfg: cfg,
- }
- id := uuid.New()
-
- mockSQS.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", id)
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
- err := svc.informExtraction(ctx, id)
- require.NoError(t, err)
-}
diff --git a/internal/document/text/service.go b/internal/document/text/service.go
index 7db2a393..81813967 100644
--- a/internal/document/text/service.go
+++ b/internal/document/text/service.go
@@ -3,14 +3,12 @@ package documenttext
import (
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/textract"
)
type ConfigProvider interface {
serviceconfig.ConfigProvider
textract.ConfigProvider
- querysync.ConfigProvider
objectstore.ConfigProvider
}
diff --git a/internal/document/text/service_test.go b/internal/document/text/service_test.go
index cdcf461e..c7a4a338 100644
--- a/internal/document/text/service_test.go
+++ b/internal/document/text/service_test.go
@@ -7,7 +7,6 @@ import (
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/aws"
"queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/textract"
"github.com/stretchr/testify/assert"
@@ -16,7 +15,6 @@ import (
type DocTextConfig struct {
aws.AWSConfig
serviceconfig.BaseConfig
- querysync.QuerySyncConfig
textract.TextractConfig
objectstore.ObjectStoreConfig
}
diff --git a/internal/document/text/storeExtract_test.go b/internal/document/text/storeExtract_test.go
index c2487a71..c446622c 100644
--- a/internal/document/text/storeExtract_test.go
+++ b/internal/document/text/storeExtract_test.go
@@ -10,7 +10,6 @@ import (
"queryorchestration/internal/database/repository"
"queryorchestration/internal/serviceconfig/objectstore"
objectstoremock "queryorchestration/mocks/objectstore"
- queuemock "queryorchestration/mocks/queue"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid"
@@ -28,9 +27,6 @@ func TestStoreExtract(t *testing.T) {
cfg := &DocTextConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
- cfg.QuerySyncURL = "querysync"
- mockSQS := queuemock.NewMockSQSClient(t)
- cfg.QueueClient = mockSQS
mockS3 := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockS3
diff --git a/internal/query/README.md b/internal/query/README.md
deleted file mode 100644
index fa0ba924..00000000
--- a/internal/query/README.md
+++ /dev/null
@@ -1,217 +0,0 @@
-# Package query
-
-## Overview
-
-Package query implements the core query processing engine for the document orchestration platform. It provides a sophisticated dependency-driven execution model where queries can extract data from documents and depend on results from other queries. The package supports multiple query types, versioning, and asynchronous processing through AWS SQS queues.
-
-## Architecture
-
-### Query Processing Model
-
-The system implements a directed acyclic graph (DAG) of query dependencies:
-1. **Independent Queries**: Process document text directly
-2. **Dependent Queries**: Use results from other queries as input
-3. **Version Management**: Each query maintains version history
-4. **Result Caching**: Processed results stored for reuse
-
-### Query Types
-
-#### JsonExtractor
-Extracts values from JSON documents using JSONPath expressions:
-- Supports nested path extraction
-- Handles arrays and complex objects
-- Returns extracted values as strings
-- Configurable default values for missing paths
-
-#### ContextFull
-Processes complete document context:
-- Access to full document text
-- Can reference other query results
-- Supports complex transformations
-- Used for advanced extraction logic
-
-### Core Components
-
-#### Service Layer (`service.go`)
-Central service orchestrating all query operations:
-- Query creation and updates
-- Version management
-- Result processing coordination
-- Dependency resolution
-
-#### Query Management
-- **create.go**: Query creation with validation
-- **update.go**: Query updates with version tracking
-- **get.go**: Query retrieval operations
-- **list.go**: Query listing with filtering
-- **normalize.go**: Configuration normalization
-- **parse.go**: Query parsing and validation
-
-#### Result Processing (`result/`)
-- **processor/**: Query type-specific processors
-- **get.go**: Result retrieval
-- **process.go**: Result computation logic
-- **set/**: Result storage operations
-- **sync/**: Result synchronization
-
-#### Synchronization
-- **sync/**: Query dependency synchronization
-- **versionsync/**: Version update propagation
-- **test/**: Query testing utilities
-
-## Query Lifecycle
-
-### 1. Query Creation
-```go
-// Create a new JSON extractor query
-query, err := queryService.Create(ctx, &CreateRequest{
- ClientID: clientID,
- Name: "extract_invoice_number",
- Type: "JsonExtractor",
- Description: "Extracts invoice number from document",
- Config: map[string]interface{}{
- "path": "$.invoice.number",
- "default": "N/A",
- },
-})
-```
-
-### 2. Query Dependencies
-```go
-// Create dependent query
-dependentQuery, err := queryService.Create(ctx, &CreateRequest{
- ClientID: clientID,
- Name: "calculate_total",
- Type: "ContextFull",
- DependsOn: []string{"extract_line_items"},
- Description: "Calculates total from line items",
- Config: config,
-})
-```
-
-### 3. Version Management
-- Each update creates a new version
-- Active version tracked separately
-- Previous versions retained for history
-- Dependency chains updated on version changes
-
-### 4. Result Processing
-```go
-// Process query for a document
-result, err := resultProcessor.Process(ctx, &ProcessRequest{
- DocumentID: documentID,
- QueryID: queryID,
-})
-```
-
-## Configuration
-
-### Query Configuration Structure
-
-#### JsonExtractor Config
-```json
-{
- "path": "$.data.field",
- "default": "default_value",
- "type": "string"
-}
-```
-
-#### ContextFull Config
-```json
-{
- "template": "Process {{.Document}} with {{.Dependencies.other_query}}",
- "format": "json",
- "validations": []
-}
-```
-
-### Service Configuration
-- Database connection for query storage
-- Queue configuration for async processing
-- Logging configuration
-- Thread pool for parallel processing
-
-## Query Processing Flow
-
-### Synchronous Processing
-1. **Validation**: Check query configuration
-2. **Dependency Resolution**: Ensure dependencies are satisfied
-3. **Execution**: Run query processor
-4. **Storage**: Save results to database
-5. **Notification**: Trigger dependent queries
-
-### Asynchronous Processing
-1. **Queue Message**: Receive processing request
-2. **Lock Acquisition**: Prevent concurrent processing
-3. **Processing**: Execute query logic
-4. **Result Storage**: Persist results
-5. **Dependency Trigger**: Queue dependent queries
-
-## Dependency Management
-
-### Dependency Resolution
-- Recursive dependency checking
-- Circular dependency prevention
-- Missing dependency handling
-- Version-aware dependency tracking
-
-### Dependency Types
-- **Hard Dependencies**: Must complete before processing
-- **Soft Dependencies**: Optional, use if available
-- **Version Dependencies**: Specific version requirements
-
-## Error Handling
-
-### Error Types
-- **Validation Errors**: Invalid configuration or input
-- **Dependency Errors**: Missing or failed dependencies
-- **Processing Errors**: Query execution failures
-- **Storage Errors**: Database operation failures
-
-### Error Recovery
-- Automatic retry for transient failures
-- Dead letter queue for persistent failures
-- Error context preservation
-- Detailed error logging
-
-## Testing
-
-### Unit Testing
-- Mock-based testing for isolated components
-- Configuration validation tests
-- Dependency resolution tests
-- Error scenario coverage
-
-### Integration Testing
-- Full query lifecycle tests
-- Dependency chain testing
-- Concurrent processing validation
-- Performance benchmarks
-
-### Test Utilities (`test/`)
-- Query creation helpers
-- Result validation utilities
-- Mock data generators
-- Test scenario builders
-
-## Performance Optimization
-
-### Caching Strategy
-- Result caching for repeated queries
-- Dependency result reuse
-- Configuration caching
-- Connection pooling
-
-### Parallel Processing
-- Concurrent query execution where possible
-- Thread pool management
-- Resource throttling
-- Queue-based load distribution
-
-### Database Optimization
-- Indexed query lookups
-- Batch result storage
-- Efficient dependency queries
-- View-based aggregations
-
diff --git a/internal/query/create.go b/internal/query/create.go
deleted file mode 100644
index 85111aed..00000000
--- a/internal/query/create.go
+++ /dev/null
@@ -1,153 +0,0 @@
-package query
-
-import (
- "context"
- "fmt"
-
- "queryorchestration/internal/database/repository"
- resultprocessor "queryorchestration/internal/query/result/processor"
- contextfull "queryorchestration/internal/query/types/contextFull"
- jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
-
- "github.com/google/uuid"
-)
-
-func (s *Service) Create(ctx context.Context, entity *resultprocessor.Create) (uuid.UUID, error) {
- err := s.normalizeCreate(ctx, entity)
- if err != nil {
- return uuid.Nil, err
- }
-
- id, err := s.submitCreate(ctx, entity)
- if err != nil {
- return uuid.Nil, err
- }
-
- return id, err
-}
-
-func (s *Service) normalizeCreate(ctx context.Context, entity *resultprocessor.Create) error {
- err := s.NormalizeQueryIDs(ctx, entity)
- if err != nil {
- return err
- }
-
- err = s.NormalizeConfig(entity)
- if err != nil {
- return err
- }
-
- validator, err := s.getCreator(entity.Type)
- if err != nil {
- return err
- }
-
- err = validator.Validate(ctx, entity)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *Service) submitCreate(ctx context.Context, entity *resultprocessor.Create) (uuid.UUID, error) {
- query, err := parseCreateQuery(entity)
- if err != nil {
- return uuid.Nil, err
- }
-
- var id uuid.UUID
- err = s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error {
- id, err = qtx.CreateQuery(ctx, query.Type)
- if err != nil {
- return err
- }
-
- version, err := qtx.AddLatestQueryVersion(ctx, id)
- if err != nil {
- return err
- }
-
- err = qtx.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
- Queryid: id,
- Versionid: version,
- })
- if err != nil {
- return err
- }
-
- if query.RequiredQueryIDs != nil {
- for _, reqQuery := range *query.RequiredQueryIDs {
- err = qtx.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
- Queryid: id,
- Requiredqueryid: reqQuery,
- Addedversion: version,
- })
- if err != nil {
- return err
- }
- }
- }
-
- if query.Config != nil {
- err = qtx.SetQueryConfig(ctx, &repository.SetQueryConfigParams{
- Queryid: id,
- Config: *query.Config,
- Addedversion: version,
- })
- if err != nil {
- return err
- }
- }
-
- return nil
- })
- if err != nil {
- return uuid.Nil, err
- }
-
- return id, nil
-
-}
-
-func (s *Service) getCreator(qType resultprocessor.Type) (resultprocessor.Creator, error) {
- switch qType {
- case resultprocessor.TypeJsonExtractor:
- return jsonextractor.NewCreator(), nil
- case resultprocessor.TypeContextFull:
- return contextfull.NewCreator(), nil
- default:
- return nil, fmt.Errorf("attempting to process invalid query type")
- }
-}
-
-type createQuery struct {
- Type repository.Querytype
- RequiredQueryIDs *[]uuid.UUID
- Config *[]byte
-}
-
-func parseCreateQuery(q *resultprocessor.Create) (*createQuery, error) {
- t, err := resultprocessor.ToDBQueryType(q.Type)
- if err != nil {
- return nil, err
- }
-
- var reqIDs *[]uuid.UUID
- if q.RequiredQueryIDs != nil {
- tIDs := *q.RequiredQueryIDs
- reqIDs = &tIDs
- }
-
- var cfg *[]byte
- if q.Config != nil {
- tC := []byte(*q.Config)
- cfg = &tC
- }
-
- return &createQuery{
- Type: t,
- RequiredQueryIDs: reqIDs,
- Config: cfg,
- }, nil
-}
diff --git a/internal/query/create_test.go b/internal/query/create_test.go
deleted file mode 100644
index b0b82ede..00000000
--- a/internal/query/create_test.go
+++ /dev/null
@@ -1,141 +0,0 @@
-package query_test
-
-import (
- "errors"
- "testing"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/query"
- resultprocessor "queryorchestration/internal/query/result/processor"
- "queryorchestration/internal/serviceconfig"
-
- "github.com/stretchr/testify/require"
-
- "github.com/google/uuid"
- "github.com/jackc/pgx/v5"
- "github.com/pashagolub/pgxmock/v3"
- "github.com/stretchr/testify/assert"
-)
-
-func TestCreate(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := query.New(cfg)
-
- config := "{\"path\":\"example_path\"}"
- q := query.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- },
- Config: &config,
- }
- create := &resultprocessor.Create{
- Type: q.Type,
- RequiredQueryIDs: q.RequiredQueryIDs,
- Config: q.Config,
- }
-
- dbType, err := resultprocessor.ToDBQueryType(create.Type)
- require.NoError(t, err)
-
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(*create.RequiredQueryIDs).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).AddRow(true),
- )
-
- pool.ExpectBeginTx(pgx.TxOptions{})
- pool.ExpectQuery("name: CreateQuery :one").WithArgs(dbType).WillReturnRows(
- pgxmock.NewRows([]string{"id"}).
- AddRow(q.ID),
- )
- pool.ExpectQuery("name: AddLatestQueryVersion :one").WithArgs(q.ID).WillReturnRows(
- pgxmock.NewRows([]string{"version"}).
- AddRow(int32(1)),
- )
- pool.ExpectExec("name: AddActiveQueryVersion :exec").WithArgs(q.ID, int32(1)).
- WillReturnResult(pgxmock.NewResult("", 1))
- for _, req := range *create.RequiredQueryIDs {
- pool.ExpectExec("name: AddRequiredQuery :exec").WithArgs(q.ID, req, int32(1)).
- WillReturnResult(pgxmock.NewResult("", 1))
- }
- pool.ExpectExec("name: SetQueryConfig :exec").WithArgs(q.ID, []byte(*create.Config), int32(1)).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectCommit()
-
- id, err := svc.Create(ctx, create)
- require.NoError(t, err)
- assert.Equal(t, q.ID, id)
-}
-
-func TestCreateMinimal(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := query.New(cfg)
-
- q := query.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeContextFull,
- }
- create := &resultprocessor.Create{
- Type: q.Type,
- }
-
- dbType, err := resultprocessor.ToDBQueryType(create.Type)
- require.NoError(t, err)
-
- pool.ExpectBeginTx(pgx.TxOptions{})
- pool.ExpectQuery("name: CreateQuery :one").WithArgs(dbType).WillReturnRows(
- pgxmock.NewRows([]string{"id"}).
- AddRow(q.ID),
- )
- pool.ExpectQuery("name: AddLatestQueryVersion :one").WithArgs(q.ID).WillReturnRows(
- pgxmock.NewRows([]string{"version"}).
- AddRow(int32(1)),
- )
- pool.ExpectExec("name: AddActiveQueryVersion :exec").WithArgs(q.ID, int32(1)).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectCommit()
-
- id, err := svc.Create(ctx, create)
- require.NoError(t, err)
- assert.Equal(t, q.ID, id)
-}
-
-func TestCreateRollback(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := query.New(cfg)
-
- ccfg := "{}"
- create := &resultprocessor.Create{
- Type: resultprocessor.TypeJsonExtractor,
- Config: &ccfg,
- }
-
- dbType, err := resultprocessor.ToDBQueryType(create.Type)
- require.NoError(t, err)
-
- pool.ExpectBeginTx(pgx.TxOptions{})
- msg := "database failure"
- pool.ExpectQuery("name: CreateQuery :one").WithArgs(dbType).WillReturnError(errors.New(msg))
- pool.ExpectRollback()
-
- _, err = svc.Create(ctx, create)
- assert.EqualError(t, err, msg)
-}
diff --git a/internal/query/createprivate_test.go b/internal/query/createprivate_test.go
deleted file mode 100644
index 9c21cf49..00000000
--- a/internal/query/createprivate_test.go
+++ /dev/null
@@ -1,196 +0,0 @@
-package query
-
-import (
- "testing"
-
- "queryorchestration/internal/database/repository"
- resultprocessor "queryorchestration/internal/query/result/processor"
- "queryorchestration/internal/serviceconfig"
-
- "github.com/google/uuid"
- "github.com/jackc/pgx/v5"
- "github.com/pashagolub/pgxmock/v3"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestGetCreator(t *testing.T) {
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg)
-
- queryType := resultprocessor.Type(resultprocessor.TypeContextFull)
- creator, err := svc.getCreator(queryType)
- require.NoError(t, err)
- assert.NotNil(t, creator)
-
- queryType = resultprocessor.Type(resultprocessor.TypeJsonExtractor)
- creator, err = svc.getCreator(queryType)
- require.NoError(t, err)
- assert.NotNil(t, creator)
-
- queryType = resultprocessor.Type(-1)
- _, err = svc.getCreator(queryType)
- assert.Error(t, err)
-}
-
-func TestParseCreateQuery(t *testing.T) {
- cfg := "{\"key\":\"value\"}"
- cQuery := &resultprocessor.Create{
- Type: resultprocessor.TypeContextFull,
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- },
- Config: &cfg,
- }
-
- resultQuery, err := parseCreateQuery(cQuery)
- require.NoError(t, err)
- rQIDs := *cQuery.RequiredQueryIDs
- qcfg := []byte(*cQuery.Config)
- assert.EqualExportedValues(t, createQuery{
- Type: repository.QuerytypeContextFull,
- RequiredQueryIDs: &rQIDs,
- Config: &qcfg,
- }, *resultQuery)
-}
-
-func TestParseCreateQueryInvalidType(t *testing.T) {
- cfg := "{\"key\":\"value\"}"
- cQuery := &resultprocessor.Create{
- Type: resultprocessor.Type(-1),
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- },
- Config: &cfg,
- }
-
- _, err := parseCreateQuery(cQuery)
- assert.EqualError(t, err, "invalid database query type")
-}
-
-func TestSubmitCreate(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg)
-
- config := "{\"path\":\"example_path\"}"
- q := Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- },
- Config: &config,
- }
- create := &resultprocessor.Create{
- Type: q.Type,
- RequiredQueryIDs: q.RequiredQueryIDs,
- Config: q.Config,
- }
-
- dbType, err := resultprocessor.ToDBQueryType(create.Type)
- require.NoError(t, err)
-
- pool.ExpectBeginTx(pgx.TxOptions{})
- pool.ExpectQuery("name: CreateQuery :one").WithArgs(dbType).WillReturnRows(
- pgxmock.NewRows([]string{"id"}).
- AddRow(q.ID),
- )
- pool.ExpectQuery("name: AddLatestQueryVersion :one").WithArgs(q.ID).WillReturnRows(
- pgxmock.NewRows([]string{"version"}).
- AddRow(int32(1)),
- )
- pool.ExpectExec("name: AddActiveQueryVersion :exec").WithArgs(q.ID, int32(1)).
- WillReturnResult(pgxmock.NewResult("", 1))
- for _, req := range *create.RequiredQueryIDs {
- pool.ExpectExec("name: AddRequiredQuery :exec").WithArgs(q.ID, req, int32(1)).
- WillReturnResult(pgxmock.NewResult("", 1))
- }
- pool.ExpectExec("name: SetQueryConfig :exec").WithArgs(q.ID, []byte(*create.Config), int32(1)).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectCommit()
-
- id, err := svc.submitCreate(ctx, create)
- require.NoError(t, err)
- assert.Equal(t, q.ID, id)
-}
-
-func TestSubmitCreateNoReqsOrConfig(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg)
-
- q := Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- }
- create := &resultprocessor.Create{
- Type: q.Type,
- }
-
- dbType, err := resultprocessor.ToDBQueryType(create.Type)
- require.NoError(t, err)
-
- pool.ExpectBeginTx(pgx.TxOptions{})
- pool.ExpectQuery("name: CreateQuery :one").WithArgs(dbType).WillReturnRows(
- pgxmock.NewRows([]string{"id"}).
- AddRow(q.ID),
- )
- pool.ExpectQuery("name: AddLatestQueryVersion :one").WithArgs(q.ID).WillReturnRows(
- pgxmock.NewRows([]string{"version"}).
- AddRow(int32(1)),
- )
- pool.ExpectExec("name: AddActiveQueryVersion :exec").WithArgs(q.ID, int32(1)).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectCommit()
-
- id, err := svc.submitCreate(ctx, create)
- require.NoError(t, err)
- assert.Equal(t, q.ID, id)
-}
-
-func TestNormalizeCreate(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg)
-
- create := &resultprocessor.Create{
- Type: resultprocessor.TypeContextFull,
- Config: nil,
- RequiredQueryIDs: &[]uuid.UUID{},
- }
-
- dbids := *create.RequiredQueryIDs
-
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(true),
- )
-
- err = svc.normalizeCreate(ctx, create)
- require.NoError(t, err)
- assert.EqualExportedValues(t, resultprocessor.Create{
- Type: resultprocessor.TypeContextFull,
- Config: nil,
- RequiredQueryIDs: nil,
- }, *create)
-}
diff --git a/internal/query/get.go b/internal/query/get.go
deleted file mode 100644
index 1b016679..00000000
--- a/internal/query/get.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package query
-
-import (
- "context"
-
- "queryorchestration/internal/database/repository"
- resultprocessor "queryorchestration/internal/query/result/processor"
-
- "github.com/google/uuid"
-)
-
-type Query struct {
- ID uuid.UUID
- Type resultprocessor.Type
- ActiveVersion int32
- LatestVersion int32
- RequiredQueryIDs *[]uuid.UUID
- Config *string
-}
-
-func (s *Service) GetWithVersion(ctx context.Context, id uuid.UUID, version int32) (*Query, error) {
- query, err := s.cfg.GetDBQueries().GetQueryWithVersion(ctx, &repository.GetQueryWithVersionParams{
- ID: &id,
- Version: &version,
- })
- if err != nil {
- return nil, err
- }
-
- return ParseQueryWithVersion(query)
-}
-
-func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Query, error) {
- query, err := s.cfg.GetDBQueries().GetQuery(ctx, id)
- if err != nil {
- return nil, err
- }
-
- return ParseFullActiveQuery(query)
-}
diff --git a/internal/query/get_test.go b/internal/query/get_test.go
deleted file mode 100644
index cb9af1e9..00000000
--- a/internal/query/get_test.go
+++ /dev/null
@@ -1,183 +0,0 @@
-package query_test
-
-import (
- "testing"
- "time"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/query"
- resultprocessor "queryorchestration/internal/query/result/processor"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/test"
-
- "github.com/google/uuid"
- "github.com/jackc/pgx/v5/pgtype"
- "github.com/pashagolub/pgxmock/v3"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestGet(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := query.New(cfg)
-
- config := "{\"path\":\"example_path\"}"
- query := query.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- ActiveVersion: int32(1),
- LatestVersion: int32(1),
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- },
- Config: &config,
- }
-
- dbReqIDs := *query.RequiredQueryIDs
-
- pool.ExpectQuery("name: GetQuery :one").WithArgs(query.ID).WillReturnRows(
- pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
- AddRow(query.ID, repository.QuerytypeJsonExtractor, query.ActiveVersion, query.LatestVersion, []byte(config), dbReqIDs),
- )
-
- returnQuery, err := svc.Get(ctx, query.ID)
- require.NoError(t, err)
-
- assert.EqualExportedValues(t, query, *returnQuery)
-}
-
-func TestGetWithVersion(t *testing.T) {
- t.Parallel()
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
- Clientid: "client_id",
- Name: "client_name",
- })
- require.NoError(t, err)
- docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{
- Clientid: "client_id",
- Hash: "hash",
- BatchID: nil,
- })
- require.NoError(t, err)
- fill := "fill"
- cleanId, err := cfg.GetDBQueries().AddDocumentClean(t.Context(), &repository.AddDocumentCleanParams{
- Documentid: docId,
- Bucket: &fill,
- Key: &fill,
- Hash: &fill,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentCleanEntry(t.Context(), &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanId,
- Version: 1,
- })
- require.NoError(t, err)
- textId, err := cfg.GetDBQueries().AddDocumentText(t.Context(), &repository.AddDocumentTextParams{
- Cleanid: cleanId,
- Bucket: fill,
- Key: fill,
- Hash: fill,
- Createdat: pgtype.Timestamp{
- Time: time.Now(),
- Valid: true,
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentTextEntry(t.Context(), &repository.AddDocumentTextEntryParams{
- Textid: textId,
- Version: 1,
- })
- require.NoError(t, err)
- contextQueryId, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeContextFull)
- require.NoError(t, err)
- contextVersion, err := cfg.GetDBQueries().AddLatestQueryVersion(t.Context(), contextQueryId)
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddActiveQueryVersion(t.Context(), &repository.AddActiveQueryVersionParams{
- Queryid: contextQueryId,
- Versionid: contextVersion,
- })
- require.NoError(t, err)
- queryId, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeJsonExtractor)
- require.NoError(t, err)
- latestVersion, err := cfg.GetDBQueries().AddLatestQueryVersion(t.Context(), queryId)
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddActiveQueryVersion(t.Context(), &repository.AddActiveQueryVersionParams{
- Queryid: queryId,
- Versionid: latestVersion,
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddRequiredQuery(t.Context(), &repository.AddRequiredQueryParams{
- Queryid: queryId,
- Requiredqueryid: contextQueryId,
- Addedversion: latestVersion,
- })
- require.NoError(t, err)
- c := `{"path":"oldkey"}`
- err = cfg.GetDBQueries().SetQueryConfig(t.Context(), &repository.SetQueryConfigParams{
- Queryid: queryId,
- Config: []byte(c),
- Addedversion: latestVersion,
- })
- require.NoError(t, err)
- strVal := `{"mykey": "example_value", "oldkey": "old_value"}`
- _, err = cfg.GetDBQueries().AddResult(t.Context(), &repository.AddResultParams{
- Queryid: contextQueryId,
- Value: strVal,
- Textentryid: textId,
- Queryversion: contextVersion,
- })
- require.NoError(t, err)
- latestVersion, err = cfg.GetDBQueries().AddLatestQueryVersion(t.Context(), queryId)
- require.NoError(t, err)
- c = `{"path":"mykey"}`
- err = cfg.GetDBQueries().SetQueryConfig(t.Context(), &repository.SetQueryConfigParams{
- Queryid: queryId,
- Config: []byte(c),
- Addedversion: latestVersion,
- })
- require.NoError(t, err)
- ctx := t.Context()
-
- svc := query.New(cfg)
-
- returnQuery, err := svc.GetWithVersion(ctx, queryId, 1)
- require.NoError(t, err)
- config := `{"path": "oldkey"}`
- assert.EqualExportedValues(t, query.Query{
- ID: queryId,
- Type: resultprocessor.TypeJsonExtractor,
- ActiveVersion: 1,
- LatestVersion: 2,
- RequiredQueryIDs: &[]uuid.UUID{
- contextQueryId,
- },
- Config: &config,
- }, *returnQuery)
-
- returnQuery, err = svc.GetWithVersion(ctx, queryId, 2)
- require.NoError(t, err)
- config = `{"path": "mykey"}`
- assert.EqualExportedValues(t, query.Query{
- ID: queryId,
- Type: resultprocessor.TypeJsonExtractor,
- ActiveVersion: 1,
- LatestVersion: 2,
- RequiredQueryIDs: &[]uuid.UUID{
- contextQueryId,
- },
- Config: &config,
- }, *returnQuery)
-}
diff --git a/internal/query/list.go b/internal/query/list.go
deleted file mode 100644
index 5f9ba98e..00000000
--- a/internal/query/list.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package query
-
-import (
- "context"
-
- "github.com/google/uuid"
-)
-
-func (s *Service) List(ctx context.Context) ([]*Query, error) {
- dbQueries, err := s.cfg.GetDBQueries().ListQueries(ctx)
- if err != nil {
- return nil, err
- }
-
- queries, err := ParseFullActiveQueryArray(dbQueries)
- if err != nil {
- return nil, err
- }
-
- return queries, nil
-}
-
-func (s *Service) ListById(ctx context.Context, ids []uuid.UUID) ([]*Query, error) {
- dbQueries, err := s.cfg.GetDBQueries().ListQueriesById(ctx, ids)
- if err != nil {
- return nil, err
- }
-
- queries, err := ParseFullActiveQueryArray(dbQueries)
- if err != nil {
- return nil, err
- }
-
- return queries, nil
-}
diff --git a/internal/query/list_test.go b/internal/query/list_test.go
deleted file mode 100644
index a9e9a514..00000000
--- a/internal/query/list_test.go
+++ /dev/null
@@ -1,86 +0,0 @@
-package query_test
-
-import (
- "testing"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/query"
- resultprocessor "queryorchestration/internal/query/result/processor"
- "queryorchestration/internal/serviceconfig"
-
- "github.com/google/uuid"
- "github.com/pashagolub/pgxmock/v3"
- "github.com/stretchr/testify/require"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestList(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := query.New(cfg)
-
- config := "{\"path\":\"example_path\"}"
- q := &query.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- ActiveVersion: int32(1),
- LatestVersion: int32(1),
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- },
- Config: &config,
- }
-
- dbReqIDs := *q.RequiredQueryIDs
-
- pool.ExpectQuery("name: ListQueries :many").WithArgs().WillReturnRows(
- pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
- AddRow(q.ID, repository.QuerytypeJsonExtractor, q.ActiveVersion, q.LatestVersion, []byte(config), dbReqIDs),
- )
-
- resList, err := svc.List(ctx)
- require.NoError(t, err)
-
- assert.EqualExportedValues(t, []*query.Query{q}, resList)
-}
-
-func TestListById(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := query.New(cfg)
-
- config := "{\"path\":\"example_path\"}"
- q := &query.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- ActiveVersion: int32(1),
- LatestVersion: int32(1),
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- },
- Config: &config,
- }
-
- dbReqIDs := *q.RequiredQueryIDs
-
- pool.ExpectQuery("name: ListQueriesById :many").WithArgs([]uuid.UUID{q.ID}).WillReturnRows(
- pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
- AddRow(q.ID, repository.QuerytypeJsonExtractor, q.ActiveVersion, q.LatestVersion, []byte(config), dbReqIDs),
- )
-
- resList, err := svc.ListById(ctx, []uuid.UUID{q.ID})
- require.NoError(t, err)
-
- assert.EqualExportedValues(t, []*query.Query{q}, resList)
-}
diff --git a/internal/query/normalize.go b/internal/query/normalize.go
deleted file mode 100644
index 6c6c6612..00000000
--- a/internal/query/normalize.go
+++ /dev/null
@@ -1,99 +0,0 @@
-package query
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "strings"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
- "queryorchestration/internal/validation"
-
- "github.com/google/uuid"
-)
-
-type Config interface {
- GetConfig() *string
- SetConfig(*string)
-}
-
-func (s *Service) NormalizeConfig(config Config) error {
- if config == nil || config.GetConfig() == nil {
- return nil
- }
-
- trim := strings.TrimSpace(*config.GetConfig())
-
- if trim == "" {
- config.SetConfig(nil)
- return nil
- }
-
- if !json.Valid([]byte(trim)) {
- return errors.New("invalid config JSON")
- }
-
- var data map[string]interface{}
- if err := json.Unmarshal([]byte(trim), &data); err != nil {
- return fmt.Errorf("error unmarshalling JSON: %w", err)
- }
-
- prettyJSON, err := json.Marshal(data)
- if err != nil {
- return fmt.Errorf("error marshaling JSON: %w", err)
- }
-
- strJSON := string(prettyJSON)
-
- config.SetConfig(&strJSON)
-
- return nil
-}
-
-type RequiredQueryIDs interface {
- GetRequiredQueryIDs() *[]uuid.UUID
- SetRequiredQueryIDs(*[]uuid.UUID)
-}
-
-func (s *Service) NormalizeQueryIDs(ctx context.Context, ids RequiredQueryIDs) error {
- if ids == nil || ids.GetRequiredQueryIDs() == nil {
- return nil
- }
-
- ide := ids.GetRequiredQueryIDs()
-
- if len(*ide) == 0 {
- ids.SetRequiredQueryIDs(nil)
- return nil
- }
-
- dedup := validation.DeduplicateArray(*ide)
- ids.SetRequiredQueryIDs(&dedup)
-
- dbids := dedup
-
- exist, err := s.cfg.GetDBQueries().AllQueriesExist(ctx, dbids)
- if err != nil {
- return err
- } else if !exist {
- return errors.New("not all required ids are present")
- }
-
- return nil
-}
-
-func (s *Service) NormalizeActiveVersion(current *Query, entity *resultprocessor.Update) error {
- if current == nil {
- return errors.New("current query required")
- } else if entity == nil {
- return nil
- }
-
- err := validation.NormalizeInClosedInterval(&entity.ActiveVersion, current.ActiveVersion, 1, current.LatestVersion+1)
- if err != nil {
- return err
- }
-
- return nil
-}
diff --git a/internal/query/normalize_test.go b/internal/query/normalize_test.go
deleted file mode 100644
index 523a2cfe..00000000
--- a/internal/query/normalize_test.go
+++ /dev/null
@@ -1,242 +0,0 @@
-package query
-
-import (
- "errors"
- "testing"
-
- "queryorchestration/internal/database/repository"
- resultprocessor "queryorchestration/internal/query/result/processor"
- "queryorchestration/internal/serviceconfig"
-
- "github.com/google/uuid"
- "github.com/pashagolub/pgxmock/v3"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestNormalizeConfig(t *testing.T) {
- s := Service{}
-
- err := s.NormalizeConfig(nil)
- require.NoError(t, err)
-
- entity := resultprocessor.Create{}
-
- entity.Config = nil
- err = s.NormalizeConfig(&entity)
- require.NoError(t, err)
- assert.Nil(t, entity.Config)
-
- cfg := ""
- entity.Config = &cfg
- err = s.NormalizeConfig(&entity)
- require.NoError(t, err)
- assert.Nil(t, entity.Config)
-
- cfg = " "
- entity.Config = &cfg
- err = s.NormalizeConfig(&entity)
- require.NoError(t, err)
- assert.Nil(t, entity.Config)
-
- cfg = "{}"
- entity.Config = &cfg
- err = s.NormalizeConfig(&entity)
- require.NoError(t, err)
- assert.Equal(t, "{}", *(entity.Config))
-
- cfg = "{\"hello\":\"bye\"}"
- entity.Config = &cfg
- err = s.NormalizeConfig(&entity)
- require.NoError(t, err)
- assert.Equal(t, "{\"hello\":\"bye\"}", *(entity.Config))
-
- cfg = " { \"hello\" : \"bye\" } "
- entity.Config = &cfg
- err = s.NormalizeConfig(&entity)
- require.NoError(t, err)
- assert.Equal(t, "{\"hello\":\"bye\"}", *(entity.Config))
-
- cfg = "{'hello':'bye'}"
- entity.Config = &cfg
- err = s.NormalizeConfig(&entity)
- assert.Error(t, err)
-
- cfg = "{\"hello\":\"}"
- entity.Config = &cfg
- err = s.NormalizeConfig(&entity)
- assert.Error(t, err)
-}
-
-func TestNormalizeQueryIDs(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- s := Service{cfg: cfg}
-
- err = s.NormalizeQueryIDs(ctx, nil)
- require.NoError(t, err)
-
- entity := resultprocessor.Create{}
-
- entity.RequiredQueryIDs = nil
- err = s.NormalizeQueryIDs(ctx, &entity)
- require.NoError(t, err)
- assert.Nil(t, entity.RequiredQueryIDs)
-
- entity.RequiredQueryIDs = &[]uuid.UUID{}
- err = s.NormalizeQueryIDs(ctx, &entity)
- require.NoError(t, err)
- assert.Nil(t, entity.RequiredQueryIDs)
-
- ids := []uuid.UUID{uuid.New()}
- entity.RequiredQueryIDs = &ids
- dbids := *entity.RequiredQueryIDs
-
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(true),
- )
-
- err = s.NormalizeQueryIDs(ctx, &entity)
- require.NoError(t, err)
- assert.ElementsMatch(t, ids, *entity.RequiredQueryIDs)
-
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(false),
- )
-
- err = s.NormalizeQueryIDs(ctx, &entity)
- assert.Error(t, err)
- assert.ElementsMatch(t, ids, *entity.RequiredQueryIDs)
-
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).
- WillReturnError(errors.New("database failure"))
-
- err = s.NormalizeQueryIDs(ctx, &entity)
- assert.Error(t, err)
- assert.ElementsMatch(t, ids, *entity.RequiredQueryIDs)
-
- singleid := uuid.New()
- entity.RequiredQueryIDs = &[]uuid.UUID{singleid, singleid}
- outids := []uuid.UUID{singleid}
- dbids = outids
-
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(true),
- )
-
- err = s.NormalizeQueryIDs(ctx, &entity)
- require.NoError(t, err)
- assert.ElementsMatch(t, outids, *entity.RequiredQueryIDs)
-}
-
-func TestNormalizeActiveVersion(t *testing.T) {
- s := Service{}
-
- t.Run("all nil", func(t *testing.T) {
- err := s.NormalizeActiveVersion(nil, nil)
- assert.Error(t, err)
- })
-
- t.Run("nil current", func(t *testing.T) {
- entity := resultprocessor.Update{}
-
- err := s.NormalizeActiveVersion(nil, &entity)
- assert.Error(t, err)
- })
-
- t.Run("nil update", func(t *testing.T) {
- current := Query{
- ActiveVersion: 2,
- LatestVersion: 4,
- }
- err := s.NormalizeActiveVersion(¤t, nil)
- require.NoError(t, err)
- })
-
- t.Run("no update", func(t *testing.T) {
- current := Query{
- ActiveVersion: 2,
- LatestVersion: 4,
- }
- entity := resultprocessor.Update{}
- err := s.NormalizeActiveVersion(¤t, &entity)
- require.NoError(t, err)
- assert.Nil(t, entity.ActiveVersion)
- })
-
- t.Run("equal update to current", func(t *testing.T) {
- current := Query{
- ActiveVersion: 2,
- LatestVersion: 4,
- }
- version := int32(2)
- entity := resultprocessor.Update{
- ActiveVersion: &version,
- }
- err := s.NormalizeActiveVersion(¤t, &entity)
- require.NoError(t, err)
- assert.Nil(t, entity.ActiveVersion)
- })
-
- t.Run("valid update", func(t *testing.T) {
- current := Query{
- ActiveVersion: 2,
- LatestVersion: 4,
- }
- version := int32(3)
- entity := resultprocessor.Update{
- ActiveVersion: &version,
- }
- err := s.NormalizeActiveVersion(¤t, &entity)
- require.NoError(t, err)
- assert.Equal(t, version, *entity.ActiveVersion)
- })
-
- t.Run("update greater than latest by 1", func(t *testing.T) {
- current := Query{
- ActiveVersion: 2,
- LatestVersion: 4,
- }
- version := int32(5)
- entity := resultprocessor.Update{
- ActiveVersion: &version,
- }
- err := s.NormalizeActiveVersion(¤t, &entity)
- require.NoError(t, err)
- assert.Equal(t, version, *entity.ActiveVersion)
- })
-
- t.Run("update greater than latest by 2", func(t *testing.T) {
- current := Query{
- ActiveVersion: 2,
- LatestVersion: 4,
- }
- version := int32(6)
- entity := resultprocessor.Update{
- ActiveVersion: &version,
- }
- err := s.NormalizeActiveVersion(¤t, &entity)
- assert.Error(t, err)
- })
-
- t.Run("update to 0", func(t *testing.T) {
- current := Query{
- ActiveVersion: 2,
- LatestVersion: 4,
- }
- version := int32(0)
- entity := resultprocessor.Update{
- ActiveVersion: &version,
- }
- err := s.NormalizeActiveVersion(¤t, &entity)
- assert.Error(t, err)
- })
-}
diff --git a/internal/query/parse.go b/internal/query/parse.go
deleted file mode 100644
index 6f5ec88d..00000000
--- a/internal/query/parse.go
+++ /dev/null
@@ -1,70 +0,0 @@
-package query
-
-import (
- "queryorchestration/internal/database/repository"
- resultprocessor "queryorchestration/internal/query/result/processor"
-
- "github.com/google/uuid"
-)
-
-func ParseQueryWithVersion(q *repository.GetQueryWithVersionRow) (*Query, error) {
- return ParseFullActiveQuery(&repository.Fullactivequery{
- Queryid: q.Queryid,
- Querytype: q.Querytype,
- Activeversion: q.Activeversion,
- Latestversion: q.Latestversion,
- Config: q.Config,
- Requiredids: q.Requiredids,
- })
-}
-
-func ParseQuery(q *Query) *resultprocessor.Query {
- return &resultprocessor.Query{
- ID: q.ID,
- Type: q.Type,
- Version: q.ActiveVersion,
- RequiredQueryIDs: q.RequiredQueryIDs,
- Config: q.Config,
- }
-}
-
-func ParseFullActiveQuery(q *repository.Fullactivequery) (*Query, error) {
- qType, err := resultprocessor.ParseDBType(q.Querytype)
- if err != nil {
- return nil, err
- }
-
- var scfg *string
- if q.Config != nil && string(q.Config) != "" {
- s := string(q.Config)
- scfg = &s
- }
-
- var reqIds *[]uuid.UUID
- if len(q.Requiredids) > 0 {
- reqIds = &q.Requiredids
- }
-
- return &Query{
- ID: q.Queryid,
- ActiveVersion: q.Activeversion,
- LatestVersion: q.Latestversion,
- Type: qType,
- RequiredQueryIDs: reqIds,
- Config: scfg,
- }, nil
-}
-
-func ParseFullActiveQueryArray(q []*repository.Fullactivequery) ([]*Query, error) {
- queries := make([]*Query, len(q))
- for index, query := range q {
- q, err := ParseFullActiveQuery(query)
- if err != nil {
- return nil, err
- }
-
- queries[index] = q
- }
-
- return queries, nil
-}
diff --git a/internal/query/parse_test.go b/internal/query/parse_test.go
deleted file mode 100644
index 14f542d9..00000000
--- a/internal/query/parse_test.go
+++ /dev/null
@@ -1,121 +0,0 @@
-package query_test
-
-import (
- "testing"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/query"
- resultprocessor "queryorchestration/internal/query/result/processor"
-
- "github.com/google/uuid"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestParseQuery(t *testing.T) {
- cfg := "example"
- q := &query.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- ActiveVersion: int32(1),
- LatestVersion: int32(2),
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- },
- Config: &cfg,
- }
-
- out := query.ParseQuery(q)
- assert.EqualExportedValues(t, resultprocessor.Query{
- ID: q.ID,
- Type: q.Type,
- Version: q.ActiveVersion,
- RequiredQueryIDs: q.RequiredQueryIDs,
- Config: q.Config,
- }, *out)
-}
-
-func TestParseFullActiveQuery(t *testing.T) {
- q := &repository.Fullactivequery{
- Queryid: uuid.New(),
- Querytype: repository.QuerytypeContextFull,
- Activeversion: int32(1),
- Latestversion: int32(2),
- Requiredids: []uuid.UUID{
- uuid.New(),
- },
- Config: []byte("example"),
- }
-
- out, err := query.ParseFullActiveQuery(q)
- require.NoError(t, err)
- bcfg := string(q.Config)
- assert.EqualExportedValues(t, query.Query{
- ID: q.Queryid,
- Type: resultprocessor.TypeContextFull,
- ActiveVersion: q.Activeversion,
- LatestVersion: q.Latestversion,
- RequiredQueryIDs: &[]uuid.UUID{
- q.Requiredids[0],
- },
- Config: &bcfg,
- }, *out)
-}
-
-func TestFullActiveQueryEmpty(t *testing.T) {
- dbQuery := &repository.Fullactivequery{
- Queryid: uuid.New(),
- Querytype: repository.QuerytypeContextFull,
- Activeversion: int32(1),
- Latestversion: int32(2),
- }
-
- out, err := query.ParseFullActiveQuery(dbQuery)
- require.NoError(t, err)
- assert.EqualExportedValues(t, query.Query{
- ID: dbQuery.Queryid,
- Type: resultprocessor.TypeContextFull,
- ActiveVersion: int32(1),
- LatestVersion: int32(2),
- }, *out)
-}
-
-func TestFullActiveQueryWithNullUUID(t *testing.T) {
- dbQuery := &repository.Fullactivequery{
- Queryid: uuid.New(),
- Querytype: repository.QuerytypeContextFull,
- Activeversion: int32(1),
- Latestversion: int32(2),
- }
-
- out, err := query.ParseFullActiveQuery(dbQuery)
- require.NoError(t, err)
- assert.EqualExportedValues(t, query.Query{
- ID: dbQuery.Queryid,
- Type: resultprocessor.TypeContextFull,
- ActiveVersion: int32(1),
- LatestVersion: int32(2),
- }, *out)
-}
-
-func TestFullActiveQueryArray(t *testing.T) {
- dbQueries := []*repository.Fullactivequery{
- {
- Queryid: uuid.New(),
- Querytype: repository.QuerytypeContextFull,
- Activeversion: int32(1),
- Latestversion: int32(2),
- },
- }
-
- out, err := query.ParseFullActiveQueryArray(dbQueries)
- require.NoError(t, err)
- assert.EqualExportedValues(t, []*query.Query{
- {
- ID: dbQueries[0].Queryid,
- Type: resultprocessor.TypeContextFull,
- ActiveVersion: int32(1),
- LatestVersion: int32(2),
- },
- }, out)
-}
diff --git a/internal/query/result/get.go b/internal/query/result/get.go
deleted file mode 100644
index 1399c231..00000000
--- a/internal/query/result/get.go
+++ /dev/null
@@ -1,49 +0,0 @@
-package result
-
-import (
- "context"
- "errors"
- "fmt"
-
- "queryorchestration/internal/database/repository"
- resultprocessor "queryorchestration/internal/query/result/processor"
- contextfull "queryorchestration/internal/query/types/contextFull"
- jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
-
- "github.com/google/uuid"
-)
-
-type GetValueWithVersionParams struct {
- Type resultprocessor.Type
- QueryID uuid.UUID
- DocumentID uuid.UUID
- QueryVersion int32
-}
-
-func (s *Service) GetValueWithVersion(ctx context.Context, params *GetValueWithVersionParams) (resultprocessor.Value, error) {
- res, err := s.cfg.GetDBQueries().GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{
- Queryid: ¶ms.QueryID,
- Queryversion: ¶ms.QueryVersion,
- Documentid: ¶ms.DocumentID,
- })
- if err != nil {
- return nil, err
- }
-
- if res.Value == nil {
- return nil, errors.New("no value found")
- }
-
- return getValueByType(params.Type, *res.Value)
-}
-
-func getValueByType(t resultprocessor.Type, value string) (resultprocessor.Value, error) {
- switch t {
- case resultprocessor.TypeJsonExtractor:
- return jsonextractor.NewResult(value), nil
- case resultprocessor.TypeContextFull:
- return contextfull.NewResult(value), nil
- default:
- return nil, fmt.Errorf("attempting to process invalid query type")
- }
-}
diff --git a/internal/query/result/get_test.go b/internal/query/result/get_test.go
deleted file mode 100644
index 4719343d..00000000
--- a/internal/query/result/get_test.go
+++ /dev/null
@@ -1,238 +0,0 @@
-package result
-
-import (
- "testing"
- "time"
-
- "queryorchestration/internal/database/repository"
- resultprocessor "queryorchestration/internal/query/result/processor"
- jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/test"
-
- "github.com/google/uuid"
- "github.com/jackc/pgx/v5/pgtype"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestGetValueByType(t *testing.T) {
- _, err := getValueByType(resultprocessor.Type(-1), "example_val")
- assert.Error(t, err)
- pro, err := getValueByType(resultprocessor.TypeContextFull, "example_context")
- require.NoError(t, err)
- assert.NotNil(t, pro)
- assert.Equal(t, "example_context", pro.GetStoreValue())
- pro, err = getValueByType(resultprocessor.TypeJsonExtractor, "example_json")
- require.NoError(t, err)
- assert.NotNil(t, pro)
- assert.Equal(t, "example_json", pro.GetStoreValue())
-}
-
-func TestGetValueWithVersion(t *testing.T) {
- t.Run("no entry", func(t *testing.T) {
- t.Parallel()
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- svc := New(cfg, &Services{})
-
- params := &GetValueWithVersionParams{
- Type: resultprocessor.TypeJsonExtractor,
- QueryID: uuid.New(),
- DocumentID: uuid.New(),
- QueryVersion: 1,
- }
-
- _, err := svc.GetValueWithVersion(t.Context(), params)
- require.EqualError(t, err, "no rows in result set")
- })
- t.Run("existing entry", func(t *testing.T) {
- t.Parallel()
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- svc := New(cfg, &Services{})
-
- err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
- Clientid: "client_id",
- Name: "client_name",
- })
- require.NoError(t, err)
- docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{
- Clientid: "client_id",
- Hash: "hash",
- })
- require.NoError(t, err)
- filetype := "pdf"
- part := uint16(1)
- key := objectstore.BucketKey{
- CreatedAt: time.Now().UTC(),
- ClientID: "clientid",
- EntityID: uuid.New(),
- Location: objectstore.Import,
- Part: &part,
- FileType: &filetype,
- }
- err = cfg.GetDBQueries().AddDocumentEntry(t.Context(), &repository.AddDocumentEntryParams{
- Documentid: docId,
- Bucket: "bucket",
- Key: key.String(),
- })
- require.NoError(t, err)
- bucket := "bucket"
- hash := "hash"
- keyStr := key.String()
- cleanId, err := cfg.GetDBQueries().AddDocumentClean(t.Context(), &repository.AddDocumentCleanParams{
- Documentid: docId,
- Bucket: &bucket,
- Hash: &hash,
- Key: &keyStr,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentCleanEntry(t.Context(), &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanId,
- Version: 1,
- })
- require.NoError(t, err)
- textId, err := cfg.GetDBQueries().AddDocumentText(t.Context(), &repository.AddDocumentTextParams{
- Cleanid: cleanId,
- Bucket: bucket,
- Hash: hash,
- Key: keyStr,
- Part: part,
- Createdat: pgtype.Timestamp{
- Valid: true,
- Time: time.Now(),
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentTextEntry(t.Context(), &repository.AddDocumentTextEntryParams{
- Textid: textId,
- Version: 1,
- })
- require.NoError(t, err)
- queryId, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeJsonExtractor)
- require.NoError(t, err)
- queryVersion, err := cfg.GetDBQueries().AddLatestQueryVersion(t.Context(), queryId)
- require.NoError(t, err)
- value := "exaple_value"
- _, err = cfg.GetDBQueries().AddResult(t.Context(), &repository.AddResultParams{
- Queryid: queryId,
- Value: value,
- Textentryid: textId,
- Queryversion: queryVersion,
- })
- require.NoError(t, err)
-
- params := &GetValueWithVersionParams{
- Type: resultprocessor.TypeJsonExtractor,
- QueryID: queryId,
- DocumentID: docId,
- QueryVersion: 1,
- }
-
- val, err := svc.GetValueWithVersion(t.Context(), params)
- require.NoError(t, err)
- v := jsonextractor.NewResult(value)
- assert.Equal(t, v, val)
- assert.Equal(t, value, v.GetStoreValue())
- })
- t.Run("not existing version", func(t *testing.T) {
- t.Parallel()
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- svc := New(cfg, &Services{})
-
- err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
- Clientid: "client_id",
- Name: "client_name",
- })
- require.NoError(t, err)
- docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{
- Clientid: "client_id",
- Hash: "hash",
- })
- require.NoError(t, err)
- filetype := "pdf"
- part := uint16(1)
- key := objectstore.BucketKey{
- CreatedAt: time.Now().UTC(),
- ClientID: "clientid",
- EntityID: uuid.New(),
- Location: objectstore.Import,
- Part: &part,
- FileType: &filetype,
- }
- err = cfg.GetDBQueries().AddDocumentEntry(t.Context(), &repository.AddDocumentEntryParams{
- Documentid: docId,
- Bucket: "bucket",
- Key: key.String(),
- })
- require.NoError(t, err)
- bucket := "bucket"
- hash := "hash"
- keyStr := key.String()
- cleanId, err := cfg.GetDBQueries().AddDocumentClean(t.Context(), &repository.AddDocumentCleanParams{
- Documentid: docId,
- Bucket: &bucket,
- Hash: &hash,
- Key: &keyStr,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentCleanEntry(t.Context(), &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanId,
- Version: 1,
- })
- require.NoError(t, err)
- textId, err := cfg.GetDBQueries().AddDocumentText(t.Context(), &repository.AddDocumentTextParams{
- Cleanid: cleanId,
- Bucket: bucket,
- Hash: hash,
- Key: keyStr,
- Part: part,
- Createdat: pgtype.Timestamp{
- Valid: true,
- Time: time.Now(),
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentTextEntry(t.Context(), &repository.AddDocumentTextEntryParams{
- Textid: textId,
- Version: 1,
- })
- require.NoError(t, err)
- queryId, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeJsonExtractor)
- require.NoError(t, err)
- queryVersion, err := cfg.GetDBQueries().AddLatestQueryVersion(t.Context(), queryId)
- require.NoError(t, err)
- value := "exaple_value"
- _, err = cfg.GetDBQueries().AddResult(t.Context(), &repository.AddResultParams{
- Queryid: queryId,
- Value: value,
- Textentryid: textId,
- Queryversion: queryVersion,
- })
- require.NoError(t, err)
-
- params := &GetValueWithVersionParams{
- Type: resultprocessor.TypeJsonExtractor,
- QueryID: queryId,
- DocumentID: docId,
- QueryVersion: 2,
- }
-
- _, err = svc.GetValueWithVersion(t.Context(), params)
- require.EqualError(t, err, "no value found")
- })
-}
diff --git a/internal/query/result/process.go b/internal/query/result/process.go
deleted file mode 100644
index 35c82570..00000000
--- a/internal/query/result/process.go
+++ /dev/null
@@ -1,115 +0,0 @@
-package result
-
-import (
- "context"
- "database/sql"
- "errors"
- "fmt"
- "log/slog"
-
- "queryorchestration/internal/database/repository"
- resultprocessor "queryorchestration/internal/query/result/processor"
- contextfull "queryorchestration/internal/query/types/contextFull"
- jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
-
- "github.com/google/uuid"
-)
-
-type Process struct {
- DocumentID uuid.UUID
- QueryID uuid.UUID
- QueryVersion int32
-}
-
-func (s *Service) Process(ctx context.Context, p *Process) (resultprocessor.Value, error) {
- slog.Debug("processing query", "query_id", p.QueryID.String(), "version", p.QueryVersion, "document_id", p.DocumentID.String())
-
- query, err := s.svc.Query.GetWithVersion(ctx, p.QueryID, p.QueryVersion)
- if err != nil {
- return nil, err
- }
-
- processQuery := &resultprocessor.Query{
- ID: query.ID,
- Version: p.QueryVersion,
- Type: query.Type,
- RequiredQueryIDs: query.RequiredQueryIDs,
- Config: query.Config,
- }
-
- values, err := s.listRequiredValues(ctx, p, processQuery)
- if err != nil {
- return nil, err
- }
-
- if s.cfg.GetLogLevel() == slog.LevelDebug {
- elength := 0
- if processQuery.RequiredQueryIDs != nil {
- elength = len(*processQuery.RequiredQueryIDs)
- }
- slog.Info("requirements", "length", len(values), "expected_length", elength)
- }
-
- processor, err := s.getProcessor(query.Type)
- if err != nil {
- return nil, err
- }
-
- val, err := processor.Process(ctx, processQuery, values)
- if err != nil {
- return nil, err
- }
-
- return getValueByType(query.Type, val)
-}
-
-func (s *Service) listRequiredValues(ctx context.Context, p *Process, query *resultprocessor.Query) ([]resultprocessor.Value, error) {
- if p == nil || query == nil || query.RequiredQueryIDs == nil || len(*query.RequiredQueryIDs) == 0 {
- return nil, nil
- }
-
- qResults, err := s.cfg.GetDBQueries().ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
- Queryid: &p.QueryID,
- Documentid: &p.DocumentID,
- Version: &p.QueryVersion,
- })
- if err != nil && !errors.Is(err, sql.ErrNoRows) {
- return nil, err
- } else if len(qResults) != len(*query.RequiredQueryIDs) {
- return nil, fmt.Errorf("required results not found for the query (%s) in document (%s)", p.QueryID, p.DocumentID)
- }
-
- slog.Debug("found results", "length", len(qResults))
-
- return parseQueryRequirementValueArray(qResults)
-}
-
-func parseQueryRequirementValueArray(v []*repository.ListQueryRequirementValuesRow) ([]resultprocessor.Value, error) {
- values := make([]resultprocessor.Value, len(v))
- for index, r := range v {
- qType, err := resultprocessor.ParseDBType(r.Querytype)
- if err != nil {
- return nil, err
- }
-
- cleanValue, err := getValueByType(qType, *r.Value)
- if err != nil {
- return nil, err
- }
-
- values[index] = cleanValue
- }
-
- return values, nil
-}
-
-func (s *Service) getProcessor(queryType resultprocessor.Type) (resultprocessor.Processor, error) {
- switch queryType {
- case resultprocessor.TypeJsonExtractor:
- return jsonextractor.NewExtractor(s.cfg), nil
- case resultprocessor.TypeContextFull:
- return contextfull.NewExtractor(), nil
- default:
- return nil, fmt.Errorf("attempting to process invalid query type")
- }
-}
diff --git a/internal/query/result/process_test.go b/internal/query/result/process_test.go
deleted file mode 100644
index 634276f1..00000000
--- a/internal/query/result/process_test.go
+++ /dev/null
@@ -1,228 +0,0 @@
-package result
-
-import (
- "testing"
- "time"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/query"
- resultprocessor "queryorchestration/internal/query/result/processor"
- jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/test"
-
- "github.com/google/uuid"
- "github.com/jackc/pgx/v5/pgtype"
- "github.com/pashagolub/pgxmock/v3"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestProcess(t *testing.T) {
- t.Parallel()
- cfg := &serviceconfig.BaseConfig{}
- test.CreateDB(t, cfg)
-
- err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
- Clientid: "client_id",
- Name: "client_name",
- })
- require.NoError(t, err)
- docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{
- Clientid: "client_id",
- Hash: "hash",
- BatchID: nil,
- Filename: nil,
- })
- require.NoError(t, err)
- fill := "fill"
- cleanId, err := cfg.GetDBQueries().AddDocumentClean(t.Context(), &repository.AddDocumentCleanParams{
- Documentid: docId,
- Bucket: &fill,
- Key: &fill,
- Hash: &fill,
- Mimetype: repository.NullCleanmimetype{
- Valid: true,
- Cleanmimetype: repository.CleanmimetypeApplicationPdf,
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentCleanEntry(t.Context(), &repository.AddDocumentCleanEntryParams{
- Cleanid: cleanId,
- Version: 1,
- })
- require.NoError(t, err)
- textId, err := cfg.GetDBQueries().AddDocumentText(t.Context(), &repository.AddDocumentTextParams{
- Cleanid: cleanId,
- Bucket: fill,
- Key: fill,
- Hash: fill,
- Createdat: pgtype.Timestamp{
- Time: time.Now(),
- Valid: true,
- },
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddDocumentTextEntry(t.Context(), &repository.AddDocumentTextEntryParams{
- Textid: textId,
- Version: 1,
- })
- require.NoError(t, err)
- contextQueryId, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeContextFull)
- require.NoError(t, err)
- contextVersion, err := cfg.GetDBQueries().AddLatestQueryVersion(t.Context(), contextQueryId)
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddActiveQueryVersion(t.Context(), &repository.AddActiveQueryVersionParams{
- Queryid: contextQueryId,
- Versionid: contextVersion,
- })
- require.NoError(t, err)
- queryId, err := cfg.GetDBQueries().CreateQuery(t.Context(), repository.QuerytypeJsonExtractor)
- require.NoError(t, err)
- latestVersion, err := cfg.GetDBQueries().AddLatestQueryVersion(t.Context(), queryId)
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddActiveQueryVersion(t.Context(), &repository.AddActiveQueryVersionParams{
- Queryid: queryId,
- Versionid: latestVersion,
- })
- require.NoError(t, err)
- err = cfg.GetDBQueries().AddRequiredQuery(t.Context(), &repository.AddRequiredQueryParams{
- Queryid: queryId,
- Requiredqueryid: contextQueryId,
- Addedversion: latestVersion,
- })
- require.NoError(t, err)
- c := `{"path":"oldkey"}`
- err = cfg.GetDBQueries().SetQueryConfig(t.Context(), &repository.SetQueryConfigParams{
- Queryid: queryId,
- Config: []byte(c),
- Addedversion: latestVersion,
- })
- require.NoError(t, err)
- strVal := `{"mykey": "example_value", "oldkey": "old_value"}`
- _, err = cfg.GetDBQueries().AddResult(t.Context(), &repository.AddResultParams{
- Queryid: contextQueryId,
- Value: strVal,
- Textentryid: textId,
- Queryversion: contextVersion,
- })
- require.NoError(t, err)
- latestVersion, err = cfg.GetDBQueries().AddLatestQueryVersion(t.Context(), queryId)
- require.NoError(t, err)
- c = `{"path": "mykey"}`
- err = cfg.GetDBQueries().SetQueryConfig(t.Context(), &repository.SetQueryConfigParams{
- Queryid: queryId,
- Config: []byte(c),
- Addedversion: latestVersion,
- })
- require.NoError(t, err)
-
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- val, err := svc.Process(t.Context(), &Process{
- DocumentID: docId,
- QueryID: queryId,
- QueryVersion: 1,
- })
- require.NoError(t, err)
- assert.NotNil(t, val)
- assert.Equal(t, "old_value", val.GetStoreValue())
-
- val, err = svc.Process(t.Context(), &Process{
- DocumentID: docId,
- QueryID: queryId,
- QueryVersion: 2,
- })
- require.NoError(t, err)
- assert.NotNil(t, val)
- assert.Equal(t, "example_value", val.GetStoreValue())
-}
-
-func TestListRequiredValue(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
-
- svc := Service{
- cfg: cfg,
- }
-
- pr, err := svc.listRequiredValues(ctx, nil, nil)
- require.NoError(t, err)
- assert.Nil(t, pr)
-
- pr, err = svc.listRequiredValues(ctx, &Process{}, &resultprocessor.Query{})
- require.NoError(t, err)
- assert.Nil(t, pr)
-
- pr, err = svc.listRequiredValues(ctx, &Process{},
- &resultprocessor.Query{
- RequiredQueryIDs: &[]uuid.UUID{},
- },
- )
- require.NoError(t, err)
- assert.Nil(t, pr)
-
- query := &resultprocessor.Query{
- ID: uuid.New(),
- Version: 2,
- RequiredQueryIDs: &[]uuid.UUID{uuid.New()},
- }
- params := Process{
- DocumentID: uuid.New(),
- QueryID: query.ID,
- QueryVersion: query.Version,
- }
-
- strVal := "axe_value"
-
- pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(&query.ID, &query.Version, ¶ms.DocumentID).
- WillReturnRows(
- pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
- AddRow(&uuid.UUID{}, (*query.RequiredQueryIDs)[0], repository.QuerytypeJsonExtractor, &strVal),
- )
-
- pr, err = svc.listRequiredValues(ctx, ¶ms, query)
- require.NoError(t, err)
- assert.ElementsMatch(t, []resultprocessor.Value{
- jsonextractor.NewResult(strVal),
- }, pr)
-}
-
-func TestGetProcessor(t *testing.T) {
- svc := Service{}
-
- pr, err := svc.getProcessor(resultprocessor.TypeJsonExtractor)
- require.NoError(t, err)
- assert.NotNil(t, pr)
-
- pr, err = svc.getProcessor(resultprocessor.TypeContextFull)
- require.NoError(t, err)
- assert.NotNil(t, pr)
-
- _, err = svc.getProcessor(resultprocessor.Type(-1))
- assert.Error(t, err)
-}
-
-func TestParseQueryRequirementValueArray(t *testing.T) {
- exval := "exampleval"
- in := []*repository.ListQueryRequirementValuesRow{
- {
- Queryid: uuid.New(),
- Value: &exval,
- Querytype: repository.QuerytypeJsonExtractor,
- },
- }
-
- out, err := parseQueryRequirementValueArray(in)
- require.NoError(t, err)
- assert.ElementsMatch(t, []resultprocessor.Value{
- jsonextractor.NewResult(exval),
- }, out)
-}
diff --git a/internal/query/result/processor/parse.go b/internal/query/result/processor/parse.go
deleted file mode 100644
index f35678a7..00000000
--- a/internal/query/result/processor/parse.go
+++ /dev/null
@@ -1,129 +0,0 @@
-package resultprocessor
-
-import (
- "fmt"
-
- "queryorchestration/internal/database/repository"
-
- "github.com/google/uuid"
-)
-
-func ParseDBNullType(qType repository.NullQuerytype) (Type, error) {
- if !qType.Valid {
- return TypeJsonExtractor, fmt.Errorf("invalid database query type")
- }
-
- return ParseDBType(qType.Querytype)
-}
-
-func ParseDBType(qType repository.Querytype) (Type, error) {
- switch qType {
- case repository.QuerytypeJsonExtractor:
- return TypeJsonExtractor, nil
- case repository.QuerytypeContextFull:
- return TypeContextFull, nil
- default:
- return TypeJsonExtractor, fmt.Errorf("invalid database query type")
- }
-}
-
-func ToDBQueryType(t Type) (repository.Querytype, error) {
- var dbType repository.Querytype
-
- switch t {
- case TypeJsonExtractor:
- dbType = repository.QuerytypeJsonExtractor
- case TypeContextFull:
- dbType = repository.QuerytypeContextFull
- default:
- return repository.QuerytypeContextFull, fmt.Errorf("invalid database query type")
- }
-
- return dbType, nil
-}
-
-func ToDBQueryTypeArray(t []Type) ([]repository.Querytype, error) {
- arr := make([]repository.Querytype, len(t))
- for index, value := range t {
- v, err := ToDBQueryType(value)
- if err != nil {
- return nil, err
- }
-
- arr[index] = v
- }
-
- return arr, nil
-}
-
-func ToDBNullQueryType(t Type) (repository.NullQuerytype, error) {
- dbType, err := ToDBQueryType(t)
- if err != nil {
- return repository.NullQuerytype{}, err
- }
-
- return repository.NullQuerytype{Querytype: dbType, Valid: true}, nil
-}
-
-func ParseDBCollectorQuery(q *repository.ListCollectorQueriesRow) (*Query, error) {
- qType, err := ParseDBNullType(q.Querytype)
- if err != nil {
- return nil, err
- }
-
- var reqIds *[]uuid.UUID
- if len(q.Requiredids) > 0 {
- reqIds = &q.Requiredids
- }
-
- return &Query{
- ID: *q.Queryid,
- Version: *q.Queryversion,
- Type: qType,
- RequiredQueryIDs: reqIds,
- }, nil
-}
-
-func ParseFullQuery(qs *repository.Fullactivequery) (*Query, error) {
- if qs == nil {
- return nil, nil
- }
-
- qt, err := ParseDBType(qs.Querytype)
- if err != nil {
- return nil, err
- }
-
- var cfg *string
- if qs.Config != nil {
- c := string(qs.Config)
- cfg = &c
- }
-
- var reqIds *[]uuid.UUID
- if len(qs.Requiredids) > 0 {
- reqIds = &qs.Requiredids
- }
-
- return &Query{
- ID: qs.Queryid,
- Type: qt,
- Version: qs.Activeversion,
- RequiredQueryIDs: reqIds,
- Config: cfg,
- }, nil
-}
-
-func ParseFullQueryArray(qs []*repository.Fullactivequery) ([]*Query, error) {
- parsed := make([]*Query, len(qs))
- for i, q := range qs {
- p, err := ParseFullQuery(q)
- if err != nil {
- return nil, err
- }
-
- parsed[i] = p
- }
-
- return parsed, nil
-}
diff --git a/internal/query/result/processor/parse_test.go b/internal/query/result/processor/parse_test.go
deleted file mode 100644
index ba0abb19..00000000
--- a/internal/query/result/processor/parse_test.go
+++ /dev/null
@@ -1,202 +0,0 @@
-package resultprocessor_test
-
-import (
- "testing"
-
- "queryorchestration/internal/database/repository"
- resultprocessor "queryorchestration/internal/query/result/processor"
-
- "github.com/google/uuid"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestParseDBCollectorQuery(t *testing.T) {
- id := "hello"
- version := int32(1)
- dbResult := repository.ListCollectorQueriesRow{
- Clientid: &id,
- Queryid: &uuid.UUID{},
- Requiredids: []uuid.UUID{},
- Querytype: repository.NullQuerytype{
- Querytype: repository.QuerytypeJsonExtractor,
- Valid: true,
- },
- Queryversion: &version,
- }
- value, err := resultprocessor.ParseDBCollectorQuery(&dbResult)
- require.NoError(t, err)
- assert.Equal(t, uuid.Nil, value.ID)
- assert.Nil(t, value.RequiredQueryIDs)
- assert.Equal(t, int32(1), value.Version)
- assert.Equal(t, resultprocessor.Type(resultprocessor.TypeJsonExtractor), value.Type)
-
- dbResult.Querytype = repository.NullQuerytype{}
- _, err = resultprocessor.ParseDBCollectorQuery(&dbResult)
- assert.EqualError(t, err, "invalid database query type")
-}
-
-func TestParseDBNullType(t *testing.T) {
- qType := repository.NullQuerytype{Valid: true, Querytype: repository.QuerytypeJsonExtractor}
- value, err := resultprocessor.ParseDBNullType(qType)
- require.NoError(t, err)
- assert.Equal(t, resultprocessor.Type(resultprocessor.TypeJsonExtractor), value)
-
- qType = repository.NullQuerytype{}
- _, err = resultprocessor.ParseDBNullType(qType)
- assert.EqualError(t, err, "invalid database query type")
-
- qType = repository.NullQuerytype{Valid: true}
- _, err = resultprocessor.ParseDBNullType(qType)
- assert.EqualError(t, err, "invalid database query type")
-}
-
-func TestParseDBType(t *testing.T) {
- qType := repository.QuerytypeJsonExtractor
- value, err := resultprocessor.ParseDBType(qType)
- require.NoError(t, err)
- assert.Equal(t, resultprocessor.Type(resultprocessor.TypeJsonExtractor), value)
-
- qType = repository.QuerytypeContextFull
- value, err = resultprocessor.ParseDBType(qType)
- require.NoError(t, err)
- assert.Equal(t, resultprocessor.Type(resultprocessor.TypeContextFull), value)
-}
-
-func TestToDBQueryType(t *testing.T) {
- dbQueryType := resultprocessor.Type(resultprocessor.TypeJsonExtractor)
- value, err := resultprocessor.ToDBQueryType(dbQueryType)
- require.NoError(t, err)
- assert.Equal(t, repository.Querytype(repository.QuerytypeJsonExtractor), value)
-
- dbQueryType = resultprocessor.Type(-1)
- _, err = resultprocessor.ToDBQueryType(dbQueryType)
- assert.EqualError(t, err, "invalid database query type")
-
- dbQueryType = resultprocessor.Type(resultprocessor.TypeContextFull)
- value, err = resultprocessor.ToDBQueryType(dbQueryType)
- require.NoError(t, err)
- assert.Equal(t, repository.Querytype(repository.QuerytypeContextFull), value)
-}
-
-func TestToDBQueryTypeArray(t *testing.T) {
- inArr := []resultprocessor.Type{
- resultprocessor.TypeJsonExtractor,
- resultprocessor.TypeContextFull,
- }
- value, err := resultprocessor.ToDBQueryTypeArray(inArr)
- require.NoError(t, err)
- assert.Equal(t, []repository.Querytype{
- repository.QuerytypeJsonExtractor,
- repository.QuerytypeContextFull,
- }, value)
-
- inArr = []resultprocessor.Type{
- resultprocessor.Type(-1),
- }
- _, err = resultprocessor.ToDBQueryTypeArray(inArr)
- assert.EqualError(t, err, "invalid database query type")
-}
-
-func TestToDBNullQueryType(t *testing.T) {
- dbQueryType := resultprocessor.Type(resultprocessor.TypeJsonExtractor)
- value, err := resultprocessor.ToDBNullQueryType(dbQueryType)
- require.NoError(t, err)
- assert.Equal(t, repository.NullQuerytype{Valid: true, Querytype: repository.QuerytypeJsonExtractor}, value)
-
- dbQueryType = resultprocessor.Type(-1)
- _, err = resultprocessor.ToDBNullQueryType(dbQueryType)
- assert.EqualError(t, err, "invalid database query type")
-
- dbQueryType = resultprocessor.Type(resultprocessor.TypeContextFull)
- value, err = resultprocessor.ToDBNullQueryType(dbQueryType)
- require.NoError(t, err)
- assert.Equal(t, repository.NullQuerytype{Valid: true, Querytype: repository.QuerytypeContextFull}, value)
-}
-
-func TestParseFullQuery(t *testing.T) {
- var q *repository.Fullactivequery
- out, err := resultprocessor.ParseFullQuery(q)
- require.NoError(t, err)
- assert.Nil(t, out)
-
- q = &repository.Fullactivequery{
- Queryid: uuid.New(),
- Querytype: repository.QuerytypeContextFull,
- Activeversion: 1,
- Latestversion: 2,
- }
-
- out, err = resultprocessor.ParseFullQuery(q)
- require.NoError(t, err)
- assert.EqualExportedValues(t, resultprocessor.Query{
- ID: q.Queryid,
- Type: resultprocessor.TypeContextFull,
- Version: 1,
- }, *out)
-
- q = &repository.Fullactivequery{
- Queryid: uuid.New(),
- Querytype: repository.QuerytypeContextFull,
- Activeversion: 1,
- Latestversion: 2,
- Requiredids: []uuid.UUID{},
- }
-
- out, err = resultprocessor.ParseFullQuery(q)
- require.NoError(t, err)
- assert.EqualExportedValues(t, &resultprocessor.Query{
- ID: q.Queryid,
- Type: resultprocessor.TypeContextFull,
- Version: 1,
- }, out)
-
- q = &repository.Fullactivequery{
- Queryid: uuid.New(),
- Querytype: repository.QuerytypeContextFull,
- Activeversion: 1,
- Latestversion: 2,
- Config: []byte("hello"),
- Requiredids: []uuid.UUID{
- uuid.New(),
- },
- }
-
- out, err = resultprocessor.ParseFullQuery(q)
- require.NoError(t, err)
- cfg := "hello"
- assert.EqualExportedValues(t, &resultprocessor.Query{
- ID: q.Queryid,
- Type: resultprocessor.TypeContextFull,
- Version: 1,
- Config: &cfg,
- RequiredQueryIDs: &[]uuid.UUID{q.Requiredids[0]},
- }, out)
-}
-
-func TestParseFullQueryArray(t *testing.T) {
- var q []*repository.Fullactivequery
- out, err := resultprocessor.ParseFullQueryArray(q)
- require.NoError(t, err)
- assert.ElementsMatch(t, []*resultprocessor.Query{}, out)
-
- q = []*repository.Fullactivequery{
- {
- Queryid: uuid.New(),
- Querytype: repository.QuerytypeContextFull,
- Activeversion: 1,
- Latestversion: 2,
- },
- }
-
- out, err = resultprocessor.ParseFullQueryArray(q)
- require.NoError(t, err)
- assert.EqualExportedValues(t, []*resultprocessor.Query{
- {
- ID: q[0].Queryid,
- Type: resultprocessor.TypeContextFull,
- Version: 1,
- },
- }, out)
-
-}
diff --git a/internal/query/result/processor/service.go b/internal/query/result/processor/service.go
deleted file mode 100644
index 6ef667b0..00000000
--- a/internal/query/result/processor/service.go
+++ /dev/null
@@ -1,84 +0,0 @@
-package resultprocessor
-
-import (
- "context"
-
- "github.com/google/uuid"
-)
-
-type Value interface {
- GetValue(ctx context.Context) (string, error)
- GetStoreValue() string
-}
-
-type Type int
-
-const (
- TypeJsonExtractor = iota
- TypeContextFull
-)
-
-type Create struct {
- Type Type
- RequiredQueryIDs *[]uuid.UUID
- Config *string
-}
-
-func (s *Create) GetConfig() *string {
- return s.Config
-}
-
-func (s *Create) SetConfig(cfg *string) {
- s.Config = cfg
-}
-
-func (s *Create) GetRequiredQueryIDs() *[]uuid.UUID {
- return s.RequiredQueryIDs
-}
-
-func (s *Create) SetRequiredQueryIDs(ids *[]uuid.UUID) {
- s.RequiredQueryIDs = ids
-}
-
-type Update struct {
- ID uuid.UUID
- ActiveVersion *int32
- RequiredQueryIDs *[]uuid.UUID
- Config *string
-}
-
-func (s *Update) GetConfig() *string {
- return s.Config
-}
-
-func (s *Update) SetConfig(cfg *string) {
- s.Config = cfg
-}
-
-func (s *Update) GetRequiredQueryIDs() *[]uuid.UUID {
- return s.RequiredQueryIDs
-}
-
-func (s *Update) SetRequiredQueryIDs(ids *[]uuid.UUID) {
- s.RequiredQueryIDs = ids
-}
-
-type Query struct {
- ID uuid.UUID
- Type Type
- Version int32
- RequiredQueryIDs *[]uuid.UUID
- Config *string
-}
-
-type Creator interface {
- Validate(ctx context.Context, entity *Create) error
-}
-
-type Updator interface {
- Validate(ctx context.Context, current *Query, entity *Update) error
-}
-
-type Processor interface {
- Process(ctx context.Context, query *Query, values []Value) (string, error)
-}
diff --git a/internal/query/result/processor/service_test.go b/internal/query/result/processor/service_test.go
deleted file mode 100644
index 7119124d..00000000
--- a/internal/query/result/processor/service_test.go
+++ /dev/null
@@ -1,94 +0,0 @@
-package resultprocessor_test
-
-import (
- "testing"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
-
- "github.com/google/uuid"
- "github.com/stretchr/testify/assert"
-)
-
-func TestCreateGetConfig(t *testing.T) {
- entity := resultprocessor.Create{}
-
- assert.Nil(t, entity.GetConfig())
-
- cfg := "example_config"
- entity.Config = &cfg
- assert.NotNil(t, entity.GetConfig())
- assert.Equal(t, cfg, *entity.GetConfig())
-}
-
-func TestCreateSetConfig(t *testing.T) {
- entity := resultprocessor.Create{}
-
- assert.Nil(t, entity.Config)
-
- cfg := "example_config"
- entity.SetConfig(&cfg)
- assert.Equal(t, cfg, *entity.Config)
-}
-
-func TestUpdateGetConfig(t *testing.T) {
- entity := resultprocessor.Update{}
-
- assert.Nil(t, entity.GetConfig())
-
- cfg := "example_config"
- entity.Config = &cfg
- assert.NotNil(t, entity.GetConfig())
- assert.Equal(t, cfg, *entity.GetConfig())
-}
-
-func TestUpdateSetConfig(t *testing.T) {
- entity := resultprocessor.Update{}
-
- assert.Nil(t, entity.Config)
-
- cfg := "example_config"
- entity.SetConfig(&cfg)
- assert.Equal(t, cfg, *entity.Config)
-}
-
-func TestCreateGetRequiredQueryIDs(t *testing.T) {
- entity := resultprocessor.Create{}
-
- assert.Nil(t, entity.GetRequiredQueryIDs())
-
- ids := []uuid.UUID{uuid.New()}
- entity.RequiredQueryIDs = &ids
- assert.NotNil(t, entity.GetRequiredQueryIDs())
- assert.Equal(t, ids, *entity.GetRequiredQueryIDs())
-}
-
-func TestCreateSetRequiredQueryIDs(t *testing.T) {
- entity := resultprocessor.Create{}
-
- assert.Nil(t, entity.RequiredQueryIDs)
-
- ids := []uuid.UUID{uuid.New()}
- entity.SetRequiredQueryIDs(&ids)
- assert.Equal(t, ids, *entity.RequiredQueryIDs)
-}
-
-func TestUpdateGetRequiredQueryIDs(t *testing.T) {
- entity := resultprocessor.Update{}
-
- assert.Nil(t, entity.GetRequiredQueryIDs())
-
- ids := []uuid.UUID{uuid.New()}
- entity.RequiredQueryIDs = &ids
- assert.NotNil(t, entity.GetRequiredQueryIDs())
- assert.Equal(t, ids, *entity.GetRequiredQueryIDs())
-}
-
-func TestUpdateSetRequiredQueryIDs(t *testing.T) {
- entity := resultprocessor.Update{}
-
- assert.Nil(t, entity.RequiredQueryIDs)
-
- ids := []uuid.UUID{uuid.New()}
- entity.SetRequiredQueryIDs(&ids)
- assert.Equal(t, ids, *entity.RequiredQueryIDs)
-}
diff --git a/internal/query/result/service.go b/internal/query/result/service.go
deleted file mode 100644
index 2d9b32b7..00000000
--- a/internal/query/result/service.go
+++ /dev/null
@@ -1,30 +0,0 @@
-package result
-
-import (
- "queryorchestration/internal/query"
- "queryorchestration/internal/serviceconfig"
-
- "github.com/google/uuid"
-)
-
-type Result struct {
- ID uuid.UUID
- QueryID uuid.UUID
- QueryVersion int32
-}
-
-type Services struct {
- Query *query.Service
-}
-
-type Service struct {
- cfg serviceconfig.ConfigProvider
- svc *Services
-}
-
-func New(cfg serviceconfig.ConfigProvider, svc *Services) *Service {
- return &Service{
- cfg,
- svc,
- }
-}
diff --git a/internal/query/result/set/service.go b/internal/query/result/set/service.go
deleted file mode 100644
index 79124659..00000000
--- a/internal/query/result/set/service.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package resultset
-
-import (
- "queryorchestration/internal/query"
- "queryorchestration/internal/query/result"
- resultsync "queryorchestration/internal/query/result/sync"
- "queryorchestration/internal/serviceconfig"
-)
-
-type Services struct {
- Result *result.Service
- Query *query.Service
- Sync *resultsync.Service
-}
-
-type Service struct {
- cfg serviceconfig.ConfigProvider
- svc *Services
-}
-
-func New(cfg serviceconfig.ConfigProvider, svc *Services) *Service {
- return &Service{
- cfg,
- svc,
- }
-}
diff --git a/internal/query/result/set/service_test.go b/internal/query/result/set/service_test.go
deleted file mode 100644
index 8cc060ab..00000000
--- a/internal/query/result/set/service_test.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package resultset_test
-
-import (
- "testing"
-
- resultset "queryorchestration/internal/query/result/set"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/serviceconfig/queue/query"
-
- "github.com/stretchr/testify/assert"
-)
-
-type QuerySyncConfig struct {
- serviceconfig.BaseConfig
- query.QueryConfig
-}
-
-func TestService(t *testing.T) {
- cfg := &QuerySyncConfig{}
- svc := resultset.New(cfg, &resultset.Services{})
- assert.NotNil(t, svc)
-}
diff --git a/internal/query/result/set/set.go b/internal/query/result/set/set.go
deleted file mode 100644
index 19bcbffa..00000000
--- a/internal/query/result/set/set.go
+++ /dev/null
@@ -1,97 +0,0 @@
-package resultset
-
-import (
- "context"
- "database/sql"
- "errors"
- "log/slog"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/query/result"
- resultprocessor "queryorchestration/internal/query/result/processor"
-
- "github.com/google/uuid"
-)
-
-type Set struct {
- DocumentID uuid.UUID `json:"document_id" validate:"required,uuid"`
- QueryID uuid.UUID `json:"query_id" validate:"required,uuid"`
-}
-
-func (s *Service) Set(ctx context.Context, params *Set) error {
- dbid := params.DocumentID
- textVersion, err := s.cfg.GetDBQueries().GetTextEntryByDocId(ctx, dbid)
- if err != nil {
- return err
- }
-
- query, err := s.svc.Query.Get(ctx, params.QueryID)
- if err != nil {
- return err
- }
-
- value, err := s.svc.Result.Process(ctx, &result.Process{
- DocumentID: params.DocumentID,
- QueryID: query.ID,
- QueryVersion: query.ActiveVersion,
- })
- if err != nil {
- return err
- }
-
- err = s.storeResult(ctx, textVersion, params, query.ActiveVersion, value)
- if err != nil {
- return err
- }
-
- slog.Debug("set query result", "query_id", params.QueryID.String(), "document_id", params.DocumentID.String())
-
- return s.informQueryDependents(ctx, params)
-}
-
-func (s *Service) storeResult(ctx context.Context, textVersion *repository.Currenttextentry, params *Set, activeVersion int32, value resultprocessor.Value) error {
- return s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
- id, err := q.AddResult(ctx, &repository.AddResultParams{
- Queryid: params.QueryID,
- Value: value.GetStoreValue(),
- Queryversion: activeVersion,
- Textentryid: textVersion.ID,
- })
- if err != nil {
- return err
- }
-
- qResults, err := q.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
- Queryid: ¶ms.QueryID,
- Documentid: ¶ms.DocumentID,
- Version: &activeVersion,
- })
- if err != nil && !errors.Is(err, sql.ErrNoRows) {
- return err
- }
-
- for _, result := range qResults {
- err = q.AddResultDependency(ctx, &repository.AddResultDependencyParams{
- Resultid: id,
- Requiredresultid: *result.ID,
- })
- if err != nil {
- return err
- }
- }
-
- return nil
- })
-}
-
-func (s *Service) informQueryDependents(ctx context.Context, params *Set) error {
- ids, err := s.cfg.GetDBQueries().ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
- Documentid: ¶ms.DocumentID,
- Queryid: ¶ms.QueryID,
- })
- if err != nil {
- return err
- }
-
- return s.svc.Sync.TriggerMultiSync(ctx, params.DocumentID, ids)
-}
diff --git a/internal/query/result/set/set_test.go b/internal/query/result/set/set_test.go
deleted file mode 100644
index ad0921c9..00000000
--- a/internal/query/result/set/set_test.go
+++ /dev/null
@@ -1,186 +0,0 @@
-package resultset
-
-import (
- "fmt"
- "testing"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/query"
- "queryorchestration/internal/query/result"
- resultprocessor "queryorchestration/internal/query/result/processor"
- resultsync "queryorchestration/internal/query/result/sync"
- "queryorchestration/internal/serviceconfig"
- queryc "queryorchestration/internal/serviceconfig/queue/query"
- queuemock "queryorchestration/mocks/queue"
-
- "github.com/stretchr/testify/require"
-
- "github.com/aws/aws-sdk-go-v2/service/sqs"
- "github.com/google/uuid"
- "github.com/pashagolub/pgxmock/v3"
- "github.com/stretchr/testify/mock"
-)
-
-type ResultSetConfig struct {
- serviceconfig.BaseConfig
- queryc.QueryConfig
-}
-
-func TestSet(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &ResultSetConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- mockSQS := queuemock.NewMockSQSClient(t)
- cfg.QueueClient = mockSQS
- cfg.QueryURL = "/i/am/here"
-
- que := query.New(cfg)
- svc := Service{
- cfg: cfg,
- svc: &Services{
- Query: que,
- Result: result.New(cfg, &result.Services{
- Query: que,
- }),
- Sync: resultsync.New(cfg),
- },
- }
-
- qcfg := "{\"path\":\"examplekey\"}"
- requiredQuery := uuid.New()
- query := &resultprocessor.Query{
- ID: uuid.New(),
- Version: 2,
- RequiredQueryIDs: &[]uuid.UUID{requiredQuery},
- Config: &qcfg,
- }
- params := Set{
- DocumentID: uuid.New(),
- QueryID: query.ID,
- }
-
- textEntryId := uuid.New()
- pool.ExpectQuery("name: GetTextEntryByDocId :one").WithArgs(params.DocumentID).
- WillReturnRows(
- pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "hash", "cleanId", "extractionVersion"}).
- AddRow(textEntryId, params.DocumentID, "buket", "/i/am/here", "example", uuid.New(), int64(543)),
- )
- pool.ExpectQuery("name: GetQuery :one").WithArgs(query.ID).WillReturnRows(
- pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
- AddRow(query.ID, repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), *query.RequiredQueryIDs),
- )
- pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(&query.ID, &query.Version).WillReturnRows(
- pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
- AddRow(query.ID, repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), *query.RequiredQueryIDs),
- )
- requiredResultId := uuid.New()
- strVal := "{\"examplekey\":\"example_value\"}"
- pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(&query.ID, &query.Version, ¶ms.DocumentID).
- WillReturnRows(
- pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
- AddRow(&requiredResultId, (*query.RequiredQueryIDs)[0], repository.QuerytypeContextFull, &strVal),
- )
- pool.ExpectBegin()
- resultId := uuid.New()
- pool.ExpectQuery("name: AddResult :one").WithArgs(query.ID, pgxmock.AnyArg(), textEntryId, query.Version).
- WillReturnRows(
- pgxmock.NewRows([]string{"id"}).
- AddRow(resultId),
- )
- pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(&query.ID, &query.Version, ¶ms.DocumentID).
- WillReturnRows(
- pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
- AddRow(&requiredResultId, (*query.RequiredQueryIDs)[0], repository.QuerytypeContextFull, &strVal),
- )
- pool.ExpectExec("name: AddResultDependency :exec").WithArgs(resultId, requiredResultId).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectCommit()
- pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(¶ms.QueryID, ¶ms.DocumentID).
- WillReturnRows(
- pgxmock.NewRows([]string{"queryId"}).
- AddRow(&requiredQuery),
- )
-
- mockSQS.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", params.DocumentID.String(), requiredQuery.String())
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
- err = svc.Set(ctx, ¶ms)
- require.NoError(t, err)
-}
-
-func TestInformQueryDependents(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &ResultSetConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- mockSQS := queuemock.NewMockSQSClient(t)
- cfg.QueueClient = mockSQS
- cfg.QueryURL = "/i/am/here"
-
- que := query.New(cfg)
- svc := Service{
- cfg: cfg,
- svc: &Services{
- Query: que,
- Result: result.New(cfg, &result.Services{
- Query: que,
- }),
- Sync: resultsync.New(cfg),
- },
- }
-
- reqOne := uuid.New()
- reqTwo := uuid.New()
- qIds := []uuid.UUID{
- reqOne,
- reqTwo,
- }
- params := &Set{
- DocumentID: uuid.New(),
- QueryID: uuid.New(),
- }
-
- pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(¶ms.QueryID, ¶ms.DocumentID).
- WillReturnRows(
- pgxmock.NewRows([]string{"queryId"}).
- AddRow(&reqOne).
- AddRow(&reqTwo),
- )
-
- mockSQS.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", params.DocumentID.String(), qIds[0].String())
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
- mockSQS.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", params.DocumentID.String(), qIds[1].String())
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
- err = svc.informQueryDependents(ctx, params)
- require.NoError(t, err)
-}
diff --git a/internal/query/result/sync/service.go b/internal/query/result/sync/service.go
deleted file mode 100644
index 716bfa54..00000000
--- a/internal/query/result/sync/service.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package resultsync
-
-import (
- "queryorchestration/internal/serviceconfig"
- queryc "queryorchestration/internal/serviceconfig/queue/query"
-)
-
-type ConfigProvider interface {
- serviceconfig.ConfigProvider
- queryc.ConfigProvider
-}
-
-type Service struct {
- cfg ConfigProvider
-}
-
-func New(cfg ConfigProvider) *Service {
- return &Service{
- cfg,
- }
-}
diff --git a/internal/query/result/sync/service_test.go b/internal/query/result/sync/service_test.go
deleted file mode 100644
index 8787131a..00000000
--- a/internal/query/result/sync/service_test.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package resultsync_test
-
-import (
- "testing"
-
- resultset "queryorchestration/internal/query/result/set"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/serviceconfig/queue/query"
-
- "github.com/stretchr/testify/assert"
-)
-
-type QuerySyncConfig struct {
- serviceconfig.BaseConfig
- query.QueryConfig
-}
-
-func TestService(t *testing.T) {
- cfg := &QuerySyncConfig{}
- svc := resultset.New(cfg, &resultset.Services{})
- assert.NotNil(t, svc)
-}
diff --git a/internal/query/result/sync/trigger.go b/internal/query/result/sync/trigger.go
deleted file mode 100644
index 1f1f2e76..00000000
--- a/internal/query/result/sync/trigger.go
+++ /dev/null
@@ -1,36 +0,0 @@
-package resultsync
-
-import (
- "context"
-
- "queryorchestration/internal/serviceconfig/queue"
-
- "github.com/google/uuid"
-)
-
-type Body struct {
- DocumentID uuid.UUID `json:"document_id" validate:"required,uuid"`
- QueryID uuid.UUID `json:"query_id" validate:"required,uuid"`
-}
-
-func (s *Service) TriggerMultiSync(ctx context.Context, documentID uuid.UUID, queryIDs []*uuid.UUID) error {
- for _, id := range queryIDs {
- err := s.TriggerSync(ctx, &Body{
- DocumentID: documentID,
- QueryID: *id,
- })
-
- if err != nil {
- return err
- }
- }
-
- return nil
-}
-
-func (s *Service) TriggerSync(ctx context.Context, params *Body) error {
- return s.cfg.SendToQueue(ctx, &queue.SendParams{
- QueueURL: s.cfg.GetQueryURL(),
- Body: params,
- })
-}
diff --git a/internal/query/result/sync/trigger_test.go b/internal/query/result/sync/trigger_test.go
deleted file mode 100644
index f3d5b138..00000000
--- a/internal/query/result/sync/trigger_test.go
+++ /dev/null
@@ -1,92 +0,0 @@
-package resultsync
-
-import (
- "fmt"
- "testing"
-
- "queryorchestration/internal/serviceconfig"
- queryc "queryorchestration/internal/serviceconfig/queue/query"
- queuemock "queryorchestration/mocks/queue"
-
- "github.com/stretchr/testify/require"
-
- "github.com/aws/aws-sdk-go-v2/service/sqs"
- "github.com/google/uuid"
- "github.com/stretchr/testify/mock"
-)
-
-type ResultSyncConfig struct {
- serviceconfig.BaseConfig
- queryc.QueryConfig
-}
-
-func TestTriggerSync(t *testing.T) {
- ctx := t.Context()
-
- cfg := &ResultSyncConfig{}
- mockSQS := queuemock.NewMockSQSClient(t)
- cfg.QueueClient = mockSQS
- cfg.QueryURL = "/i/am/here"
-
- svc := Service{
- cfg: cfg,
- }
-
- params := &Body{
- DocumentID: uuid.New(),
- QueryID: uuid.New(),
- }
-
- mockSQS.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", params.DocumentID.String(), params.QueryID.String())
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
- err := svc.TriggerSync(ctx, params)
- require.NoError(t, err)
-}
-
-func TestTriggerMultiSync(t *testing.T) {
- ctx := t.Context()
-
- cfg := &ResultSyncConfig{}
- mockSQS := queuemock.NewMockSQSClient(t)
- cfg.QueueClient = mockSQS
- cfg.QueryURL = "/i/am/here"
-
- svc := Service{
- cfg: cfg,
- }
-
- docId := uuid.New()
- queryIDOne := uuid.New()
- queryIDTwo := uuid.New()
-
- mockSQS.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", docId.String(), queryIDOne.String())
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
- mockSQS.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", docId.String(), queryIDTwo.String())
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
- err := svc.TriggerMultiSync(ctx, docId, []*uuid.UUID{&queryIDOne, &queryIDTwo})
- require.NoError(t, err)
-}
diff --git a/internal/query/service.go b/internal/query/service.go
deleted file mode 100644
index 8d52f772..00000000
--- a/internal/query/service.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package query
-
-import (
- "queryorchestration/internal/serviceconfig"
-)
-
-type Service struct {
- cfg serviceconfig.ConfigProvider
-}
-
-func New(cfg serviceconfig.ConfigProvider) *Service {
- return &Service{
- cfg,
- }
-}
diff --git a/internal/query/sync/service.go b/internal/query/sync/service.go
deleted file mode 100644
index a8bbe1b2..00000000
--- a/internal/query/sync/service.go
+++ /dev/null
@@ -1,28 +0,0 @@
-package querysync
-
-import (
- resultsync "queryorchestration/internal/query/result/sync"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/serviceconfig/queue/query"
-)
-
-type ConfigProvider interface {
- serviceconfig.ConfigProvider
- query.ConfigProvider
-}
-
-type Services struct {
- ResultSync *resultsync.Service
-}
-
-type Service struct {
- cfg ConfigProvider
- svc *Services
-}
-
-func New(cfg ConfigProvider, svc *Services) *Service {
- return &Service{
- cfg,
- svc,
- }
-}
diff --git a/internal/query/sync/sync.go b/internal/query/sync/sync.go
deleted file mode 100644
index 59929917..00000000
--- a/internal/query/sync/sync.go
+++ /dev/null
@@ -1,27 +0,0 @@
-package querysync
-
-import (
- "context"
- "log/slog"
-
- "github.com/google/uuid"
-)
-
-func (s *Service) Sync(ctx context.Context, documentId uuid.UUID) error {
- isextracted, err := s.cfg.GetDBQueries().IsDocumentTextExtracted(ctx, documentId)
- if err != nil {
- return err
- } else if !isextracted {
- slog.Error("document has no extracted text", "id", documentId)
- return nil
- }
-
- unsyncedQueries, err := s.cfg.GetDBQueries().ListUnsyncedNoDepsQueriesByDocId(ctx, &documentId)
- if err != nil {
- return err
- }
-
- slog.Debug("unsynced queries", "document_id", documentId.String(), "queries", unsyncedQueries)
-
- return s.svc.ResultSync.TriggerMultiSync(ctx, documentId, unsyncedQueries)
-}
diff --git a/internal/query/sync/sync_test.go b/internal/query/sync/sync_test.go
deleted file mode 100644
index 9b50e024..00000000
--- a/internal/query/sync/sync_test.go
+++ /dev/null
@@ -1,93 +0,0 @@
-package querysync
-
-import (
- "fmt"
- "testing"
-
- "queryorchestration/internal/database/repository"
- resultsync "queryorchestration/internal/query/result/sync"
- "queryorchestration/internal/serviceconfig"
- queryc "queryorchestration/internal/serviceconfig/queue/query"
- queuemock "queryorchestration/mocks/queue"
-
- "github.com/aws/aws-sdk-go-v2/service/sqs"
- "github.com/google/uuid"
- "github.com/pashagolub/pgxmock/v3"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
- "github.com/stretchr/testify/require"
-)
-
-type QuerySyncConfig struct {
- serviceconfig.BaseConfig
- queryc.QueryConfig
-}
-
-func TestSync(t *testing.T) {
- ctx := t.Context()
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QuerySyncConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- mockSQS := queuemock.NewMockSQSClient(t)
- cfg.QueueClient = mockSQS
- cfg.QueryURL = "/i/am/here"
- svc := Service{
- cfg: cfg,
- svc: &Services{
- ResultSync: resultsync.New(cfg),
- },
- }
-
- t.Run("valid", func(t *testing.T) {
- id := uuid.New()
- reqOne := uuid.New()
- reqTwo := uuid.New()
-
- pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(id).
- WillReturnRows(
- pgxmock.NewRows([]string{"isextracted"}).
- AddRow(true),
- )
- pool.ExpectQuery("name: ListUnsyncedNoDepsQueriesByDocId :many").WithArgs(&id).
- WillReturnRows(
- pgxmock.NewRows([]string{"id"}).
- AddRow(&reqOne).
- AddRow(&reqTwo),
- )
-
- mockSQS.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", id.String(), reqOne.String())
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
- mockSQS.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", id.String(), reqTwo.String())
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
- assert.Nil(t, svc.Sync(ctx, id))
- })
- t.Run("not extracted", func(t *testing.T) {
- id := uuid.New()
-
- pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(id).
- WillReturnRows(
- pgxmock.NewRows([]string{"isextracted"}).
- AddRow(false),
- )
-
- assert.Nil(t, svc.Sync(ctx, id))
- })
-}
diff --git a/internal/query/test/service.go b/internal/query/test/service.go
deleted file mode 100644
index 99085cc8..00000000
--- a/internal/query/test/service.go
+++ /dev/null
@@ -1,26 +0,0 @@
-package querytest
-
-import (
- "queryorchestration/internal/collector"
- "queryorchestration/internal/document"
- "queryorchestration/internal/query/result"
- "queryorchestration/internal/serviceconfig"
-)
-
-type Services struct {
- Result *result.Service
- Collector *collector.Service
- Document *document.Service
-}
-
-type Service struct {
- cfg serviceconfig.ConfigProvider
- svc *Services
-}
-
-func New(cfg serviceconfig.ConfigProvider, svc *Services) *Service {
- return &Service{
- cfg,
- svc,
- }
-}
diff --git a/internal/query/test/test.go b/internal/query/test/test.go
deleted file mode 100644
index 254d6984..00000000
--- a/internal/query/test/test.go
+++ /dev/null
@@ -1,29 +0,0 @@
-package querytest
-
-import (
- "context"
-
- "queryorchestration/internal/query/result"
-)
-
-func (s *Service) Test(ctx context.Context, filters result.Process) (string, error) {
- res, err := s.svc.Result.GetValueWithVersion(ctx, &result.GetValueWithVersionParams{
- QueryID: filters.QueryID,
- DocumentID: filters.DocumentID,
- QueryVersion: filters.QueryVersion,
- })
- if err == nil {
- return res.GetValue(ctx)
- }
-
- value, err := s.svc.Result.Process(ctx, &result.Process{
- DocumentID: filters.DocumentID,
- QueryID: filters.QueryID,
- QueryVersion: filters.QueryVersion,
- })
- if err != nil {
- return "", err
- }
-
- return value.GetValue(ctx)
-}
diff --git a/internal/query/test/test_test.go b/internal/query/test/test_test.go
deleted file mode 100644
index d0ed50d7..00000000
--- a/internal/query/test/test_test.go
+++ /dev/null
@@ -1,67 +0,0 @@
-package querytest_test
-
-import (
- "testing"
-
- "queryorchestration/internal/collector"
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/document"
- "queryorchestration/internal/query"
- "queryorchestration/internal/query/result"
- querytest "queryorchestration/internal/query/test"
- "queryorchestration/internal/serviceconfig"
-
- "github.com/stretchr/testify/require"
-
- "github.com/google/uuid"
- "github.com/pashagolub/pgxmock/v3"
- "github.com/stretchr/testify/assert"
-)
-
-func TestTest(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- col := collector.New(cfg)
- docsvc := document.New(cfg)
- svc := querytest.New(cfg, &querytest.Services{
- Document: docsvc,
- Collector: col,
- Result: result.New(cfg, &result.Services{
- Query: query.New(cfg),
- }),
- })
-
- doc := document.DocumentSummary{
- ID: uuid.New(),
- }
- params := &result.Process{
- QueryID: uuid.New(),
- DocumentID: doc.ID,
- QueryVersion: int32(1),
- }
-
- reqID := uuid.New()
- pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(¶ms.QueryID, ¶ms.QueryVersion).WillReturnRows(
- pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
- AddRow(params.QueryID, repository.QuerytypeJsonExtractor, int32(1), params.QueryVersion+1, []byte("{\"path\":\"oldkey\"}"), []uuid.UUID{reqID}),
- )
- strVal := "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}"
- pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(¶ms.QueryID, ¶ms.QueryVersion, ¶ms.DocumentID).
- WillReturnRows(
- pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
- AddRow(&uuid.UUID{}, reqID, repository.QuerytypeContextFull, &strVal),
- )
- pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(params.QueryID).WillReturnRows(
- pgxmock.NewRows([]string{"config"}).
- AddRow([]byte("{\"path\":\"oldkey\"}")),
- )
-
- result, err := svc.Test(ctx, *params)
- require.NoError(t, err)
- assert.Equal(t, "old_value", result)
-}
diff --git a/internal/query/types/contextFull/creator.go b/internal/query/types/contextFull/creator.go
deleted file mode 100644
index c445756f..00000000
--- a/internal/query/types/contextFull/creator.go
+++ /dev/null
@@ -1,18 +0,0 @@
-package contextfull
-
-import (
- "context"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
-)
-
-type Creator struct {
-}
-
-func NewCreator() *Creator {
- return &Creator{}
-}
-
-func (s *Creator) Validate(ctx context.Context, entity *resultprocessor.Create) error {
- return nil
-}
diff --git a/internal/query/types/contextFull/creator_test.go b/internal/query/types/contextFull/creator_test.go
deleted file mode 100644
index 4e3e9722..00000000
--- a/internal/query/types/contextFull/creator_test.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package contextfull_test
-
-import (
- "testing"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
- contextfull "queryorchestration/internal/query/types/contextFull"
-
- "github.com/stretchr/testify/require"
-)
-
-func TestCreatorValidate(t *testing.T) {
- svc := contextfull.NewCreator()
-
- entity := &resultprocessor.Create{
- Type: resultprocessor.TypeContextFull,
- }
-
- err := svc.Validate(t.Context(), entity)
- require.NoError(t, err)
-}
diff --git a/internal/query/types/contextFull/process_test.go b/internal/query/types/contextFull/process_test.go
deleted file mode 100644
index 4210845d..00000000
--- a/internal/query/types/contextFull/process_test.go
+++ /dev/null
@@ -1,37 +0,0 @@
-package contextfull_test
-
-import (
- "testing"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
- contextfull "queryorchestration/internal/query/types/contextFull"
-
- "github.com/google/uuid"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-)
-
-func TestContextFull(t *testing.T) {
- ctx := t.Context()
-
- extractor := contextfull.NewExtractor()
-
- query := &resultprocessor.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- Version: int32(1),
- }
-
- values := []resultprocessor.Value{}
-
- value, err := extractor.Process(ctx, query, values)
- require.NoError(t, err)
- assert.Equal(t, `{"keyone":"valueone","keytwo":"valuetwo"}`, value)
-
- values = []resultprocessor.Value{
- contextfull.NewResult("example_result"),
- }
-
- _, err = extractor.Process(ctx, query, values)
- assert.EqualError(t, err, "no requirements expected")
-}
diff --git a/internal/query/types/contextFull/result.go b/internal/query/types/contextFull/result.go
deleted file mode 100644
index 8ac7f279..00000000
--- a/internal/query/types/contextFull/result.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package contextfull
-
-import (
- "context"
-)
-
-type Result struct {
- value string
-}
-
-func NewResult(value string) *Result {
- return &Result{value}
-}
-
-func (r *Result) GetValue(ctx context.Context) (string, error) {
- return r.value, nil
-}
-
-func (r Result) GetStoreValue() string {
- return r.value
-}
diff --git a/internal/query/types/contextFull/service.go b/internal/query/types/contextFull/service.go
deleted file mode 100644
index 1d4ea70b..00000000
--- a/internal/query/types/contextFull/service.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package contextfull
-
-import (
- "context"
- "errors"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
-)
-
-type Extractor struct {
-}
-
-func NewExtractor() *Extractor {
- return &Extractor{}
-}
-
-func (e *Extractor) Process(ctx context.Context, query *resultprocessor.Query, values []resultprocessor.Value) (string, error) {
- if len(values) > 0 {
- return "", errors.New("no requirements expected")
- }
-
- return `{"keyone":"valueone","keytwo":"valuetwo"}`, nil
-}
diff --git a/internal/query/types/contextFull/updator.go b/internal/query/types/contextFull/updator.go
deleted file mode 100644
index 74adcdf6..00000000
--- a/internal/query/types/contextFull/updator.go
+++ /dev/null
@@ -1,17 +0,0 @@
-package contextfull
-
-import (
- "context"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
-)
-
-type Updator struct{}
-
-func NewUpdator() *Updator {
- return &Updator{}
-}
-
-func (s *Updator) Validate(ctx context.Context, current *resultprocessor.Query, entity *resultprocessor.Update) error {
- return nil
-}
diff --git a/internal/query/types/contextFull/updator_test.go b/internal/query/types/contextFull/updator_test.go
deleted file mode 100644
index c6c82fe2..00000000
--- a/internal/query/types/contextFull/updator_test.go
+++ /dev/null
@@ -1,28 +0,0 @@
-package contextfull_test
-
-import (
- "testing"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
- contextfull "queryorchestration/internal/query/types/contextFull"
-
- "github.com/google/uuid"
- "github.com/stretchr/testify/require"
-)
-
-func TestUpdatorValidate(t *testing.T) {
- svc := contextfull.NewUpdator()
-
- current := &resultprocessor.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeContextFull,
- Version: int32(1),
- }
-
- entity := &resultprocessor.Update{
- ID: current.ID,
- }
-
- err := svc.Validate(t.Context(), current, entity)
- require.NoError(t, err)
-}
diff --git a/internal/query/types/jsonExtractor/creator.go b/internal/query/types/jsonExtractor/creator.go
deleted file mode 100644
index b9b2f609..00000000
--- a/internal/query/types/jsonExtractor/creator.go
+++ /dev/null
@@ -1,22 +0,0 @@
-package jsonextractor
-
-import (
- "context"
- "errors"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
-)
-
-type Creator struct{}
-
-func NewCreator() *Creator {
- return &Creator{}
-}
-
-func (s *Creator) Validate(ctx context.Context, entity *resultprocessor.Create) error {
- if entity.Config == nil {
- return errors.New("config required")
- }
-
- return nil
-}
diff --git a/internal/query/types/jsonExtractor/creator_test.go b/internal/query/types/jsonExtractor/creator_test.go
deleted file mode 100644
index 047771f1..00000000
--- a/internal/query/types/jsonExtractor/creator_test.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package jsonextractor_test
-
-import (
- "testing"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
- jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
-
- "github.com/stretchr/testify/require"
-)
-
-func TestCreatorValidate(t *testing.T) {
- svc := jsonextractor.NewCreator()
-
- ccfg := "{}"
- entity := &resultprocessor.Create{
- Type: resultprocessor.TypeJsonExtractor,
- Config: &ccfg,
- }
-
- err := svc.Validate(t.Context(), entity)
- require.NoError(t, err)
-}
diff --git a/internal/query/types/jsonExtractor/process_test.go b/internal/query/types/jsonExtractor/process_test.go
deleted file mode 100644
index daec7bab..00000000
--- a/internal/query/types/jsonExtractor/process_test.go
+++ /dev/null
@@ -1,163 +0,0 @@
-package jsonextractor_test
-
-import (
- "fmt"
- "testing"
-
- "queryorchestration/internal/database/repository"
- resultprocessor "queryorchestration/internal/query/result/processor"
- contextfull "queryorchestration/internal/query/types/contextFull"
- jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
- "queryorchestration/internal/serviceconfig"
-
- "github.com/stretchr/testify/require"
-
- "github.com/google/uuid"
- "github.com/pashagolub/pgxmock/v3"
- "github.com/stretchr/testify/assert"
-)
-
-func TestJSONProcess(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &serviceconfig.BaseConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
-
- extractor := jsonextractor.NewExtractor(cfg)
-
- config := "{\"path\":\"key\"}"
- query := &resultprocessor.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- Version: int32(1),
- Config: &config,
- }
- entryValue := "value"
-
- jsonString := fmt.Sprintf("{\"key\": \"%s\"}", entryValue)
- values := []resultprocessor.Value{
- contextfull.NewResult(jsonString),
- }
-
- value, err := extractor.Process(ctx, query, values)
- require.NoError(t, err)
- assert.Equal(t, entryValue, value)
-
- entryValue = ""
- jsonString = fmt.Sprintf("{\"key\": \"%s\"}", entryValue)
- values = []resultprocessor.Value{
- contextfull.NewResult(jsonString),
- }
- pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(query.ID).
- WillReturnRows(
- pgxmock.NewRows([]string{"config"}).
- AddRow([]byte(config)),
- )
-
- value, err = extractor.Process(ctx, query, values)
- require.NoError(t, err)
- assert.Equal(t, entryValue, value)
-
- entryValue = "1"
- jsonString = fmt.Sprintf("{\"key\": %s", entryValue)
- values = []resultprocessor.Value{
- contextfull.NewResult(jsonString),
- }
- pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(query.ID).
- WillReturnRows(
- pgxmock.NewRows([]string{"config"}).
- AddRow([]byte(config)),
- )
-
- value, err = extractor.Process(ctx, query, values)
- require.NoError(t, err)
- assert.Equal(t, entryValue, value)
-}
-
-func TestJSONProcessJSON(t *testing.T) {
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
-
- extractor := jsonextractor.NewExtractor(cfg)
-
- query := &resultprocessor.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- Version: int32(1),
- }
- entryValue := "value"
-
- jsonString := fmt.Sprintf("{\"key\": \"%s\"}", entryValue)
- values := []resultprocessor.Value{
- contextfull.NewResult(jsonString),
- }
-
- value, err := extractor.Process(ctx, query, values)
- assert.EqualError(t, err, "config required")
- assert.Empty(t, value)
-
- config := "{\"path\":\"invalid_key\"}"
- query.Config = &config
-
- value, err = extractor.Process(ctx, query, values)
- assert.EqualError(t, err, "JSON path does not exist: invalid_key")
- assert.Empty(t, value)
-
- config = ""
-
- value, err = extractor.Process(ctx, query, values)
- assert.EqualError(t, err, "unexpected end of JSON input")
- assert.Empty(t, value)
-
- config = "{\"path\":\"\"}"
-
- value, err = extractor.Process(ctx, query, values)
- assert.EqualError(t, err, "JSON path does not exist: ")
- assert.Empty(t, value)
-
- config = "{\"path\":}"
-
- value, err = extractor.Process(ctx, query, values)
- assert.EqualError(t, err, "invalid character '}' looking for beginning of value")
- assert.Empty(t, value)
-}
-
-func TestJSONProcessResults(t *testing.T) {
- ctx := t.Context()
-
- cfg := &serviceconfig.BaseConfig{}
-
- extractor := jsonextractor.NewExtractor(cfg)
-
- query := &resultprocessor.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- Version: int32(1),
- }
-
- results := []resultprocessor.Value{}
- value, err := extractor.Process(ctx, query, results)
- assert.EqualError(t, err, "JSON Extraction requires 1 result")
- assert.Empty(t, value)
-
- results = []resultprocessor.Value{
- contextfull.NewResult(""),
- contextfull.NewResult(""),
- }
- value, err = extractor.Process(ctx, query, results)
- assert.EqualError(t, err, "JSON Extraction requires 1 result")
- assert.Empty(t, value)
-
- results = []resultprocessor.Value{
- contextfull.NewResult(""),
- contextfull.NewResult(""),
- contextfull.NewResult(""),
- }
- value, err = extractor.Process(ctx, query, results)
- assert.EqualError(t, err, "JSON Extraction requires 1 result")
- assert.Empty(t, value)
-}
diff --git a/internal/query/types/jsonExtractor/result.go b/internal/query/types/jsonExtractor/result.go
deleted file mode 100644
index 08b8a510..00000000
--- a/internal/query/types/jsonExtractor/result.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package jsonextractor
-
-import (
- "context"
-)
-
-type Result struct {
- value string
-}
-
-func NewResult(value string) *Result {
- return &Result{value}
-}
-
-func (r *Result) GetValue(ctx context.Context) (string, error) {
- return r.value, nil
-}
-
-func (r Result) GetStoreValue() string {
- return r.value
-}
diff --git a/internal/query/types/jsonExtractor/service.go b/internal/query/types/jsonExtractor/service.go
deleted file mode 100644
index 6145ef4c..00000000
--- a/internal/query/types/jsonExtractor/service.go
+++ /dev/null
@@ -1,55 +0,0 @@
-package jsonextractor
-
-import (
- "context"
- "encoding/json"
- "fmt"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
- "queryorchestration/internal/serviceconfig"
-
- "github.com/tidwall/gjson"
-)
-
-type Extractor struct {
- cfg serviceconfig.ConfigProvider
-}
-
-type Config struct {
- Path string `json:"path"`
-}
-
-func NewExtractor(cfg serviceconfig.ConfigProvider) *Extractor {
- return &Extractor{cfg}
-}
-
-func (e *Extractor) Process(ctx context.Context, query *resultprocessor.Query, values []resultprocessor.Value) (string, error) {
- if len(values) != 1 {
- return "", fmt.Errorf("JSON Extraction requires 1 result")
- } else if query.Config == nil {
- return "", fmt.Errorf("config required")
- }
-
- value, err := values[0].GetValue(ctx)
- if err != nil {
- return "", err
- }
-
- var config Config
- err = json.Unmarshal([]byte(*query.Config), &config)
- if err != nil {
- return "", err
- }
-
- return e.extract(value, config.Path)
-}
-
-func (e *Extractor) extract(jsonString string, path string) (string, error) {
- value := gjson.Get(jsonString, path)
-
- if !value.Exists() {
- return "", fmt.Errorf("JSON path does not exist: %s", path)
- }
-
- return value.String(), nil
-}
diff --git a/internal/query/types/jsonExtractor/updator.go b/internal/query/types/jsonExtractor/updator.go
deleted file mode 100644
index 17acbc37..00000000
--- a/internal/query/types/jsonExtractor/updator.go
+++ /dev/null
@@ -1,18 +0,0 @@
-package jsonextractor
-
-import (
- "context"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
-)
-
-type Updator struct {
-}
-
-func NewUpdator() *Updator {
- return &Updator{}
-}
-
-func (s *Updator) Validate(ctx context.Context, current *resultprocessor.Query, entity *resultprocessor.Update) error {
- return nil
-}
diff --git a/internal/query/types/jsonExtractor/updator_test.go b/internal/query/types/jsonExtractor/updator_test.go
deleted file mode 100644
index b450ecd0..00000000
--- a/internal/query/types/jsonExtractor/updator_test.go
+++ /dev/null
@@ -1,28 +0,0 @@
-package jsonextractor_test
-
-import (
- "testing"
-
- resultprocessor "queryorchestration/internal/query/result/processor"
- jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
-
- "github.com/google/uuid"
- "github.com/stretchr/testify/require"
-)
-
-func TestUpdatorValidate(t *testing.T) {
- svc := jsonextractor.NewUpdator()
-
- current := &resultprocessor.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- Version: int32(1),
- }
-
- entity := &resultprocessor.Update{
- ID: current.ID,
- }
-
- err := svc.Validate(t.Context(), current, entity)
- require.NoError(t, err)
-}
diff --git a/internal/query/update/service.go b/internal/query/update/service.go
deleted file mode 100644
index 3804fa7b..00000000
--- a/internal/query/update/service.go
+++ /dev/null
@@ -1,28 +0,0 @@
-package queryupdate
-
-import (
- "queryorchestration/internal/query"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/serviceconfig/queue/queryversionsync"
-)
-
-type ConfigProvider interface {
- serviceconfig.ConfigProvider
- queryversionsync.ConfigProvider
-}
-
-type Services struct {
- Query *query.Service
-}
-
-type Service struct {
- cfg ConfigProvider
- svc *Services
-}
-
-func New(cfg ConfigProvider, svc *Services) *Service {
- return &Service{
- cfg,
- svc,
- }
-}
diff --git a/internal/query/update/update.go b/internal/query/update/update.go
deleted file mode 100644
index 90f949cb..00000000
--- a/internal/query/update/update.go
+++ /dev/null
@@ -1,236 +0,0 @@
-package queryupdate
-
-import (
- "context"
- "errors"
- "fmt"
- "log/slog"
-
- queryversionsyncrunner "queryorchestration/api/queryVersionSyncRunner"
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/query"
- resultprocessor "queryorchestration/internal/query/result/processor"
- contextfull "queryorchestration/internal/query/types/contextFull"
- jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
- "queryorchestration/internal/serviceconfig/queue"
- "queryorchestration/internal/validation"
-
- "github.com/google/uuid"
-)
-
-func (s *Service) Update(ctx context.Context, entity *resultprocessor.Update) error {
- current, err := s.svc.Query.Get(ctx, entity.ID)
- if err != nil {
- return err
- }
-
- err = s.normalizeUpdate(ctx, current, entity)
- if err != nil {
- return err
- }
-
- err = s.submitUpdate(ctx, current, entity)
- if err != nil {
- return err
- }
-
- err = s.informUpdate(ctx, entity)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *Service) informUpdate(ctx context.Context, entity *resultprocessor.Update) error {
- if entity.ActiveVersion == nil {
- return nil
- }
-
- return s.cfg.SendToQueue(ctx, &queue.SendParams{
- QueueURL: s.cfg.GetQueryVersionSyncURL(),
- Body: queryversionsyncrunner.Body{
- QueryID: entity.ID,
- },
- })
-}
-
-func (s *Service) normalizeUpdateRequiredQueryIDs(ctx context.Context, current *query.Query, entity query.RequiredQueryIDs) error {
- err := s.svc.Query.NormalizeQueryIDs(ctx, entity)
- if err != nil {
- return err
- }
-
- if entity.GetRequiredQueryIDs() == nil {
- return nil
- }
-
- createsloop, err := s.cfg.GetDBQueries().IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
- Queryid: ¤t.ID,
- Requiredqueryids: *entity.GetRequiredQueryIDs(),
- })
- if err != nil {
- return err
- } else if createsloop {
- return errors.New("required ids create a loop")
- }
-
- addIDs := getSetDifference(entity.GetRequiredQueryIDs(), current.RequiredQueryIDs)
- removeIDs := getSetDifference(current.RequiredQueryIDs, entity.GetRequiredQueryIDs())
-
- if len(addIDs) == 0 && len(removeIDs) == 0 {
- entity.SetRequiredQueryIDs(nil)
- }
-
- return nil
-}
-
-func (s *Service) normalizeUpdate(ctx context.Context, current *query.Query, entity *resultprocessor.Update) error {
- err := s.svc.Query.NormalizeActiveVersion(current, entity)
- if err != nil {
- return err
- }
-
- err = s.normalizeUpdateRequiredQueryIDs(ctx, current, entity)
- if err != nil {
- return err
- }
-
- err = s.svc.Query.NormalizeConfig(entity)
- if err != nil {
- return err
- }
-
- activeVersionName, err := validation.GetFieldName(entity, entity.ActiveVersion)
- if err != nil {
- return err
- }
-
- if (entity.ActiveVersion == nil || *entity.ActiveVersion == current.LatestVersion+1) &&
- validation.AreAllPointersNilExcept(entity, activeVersionName) {
- return errors.New("no changes")
- }
-
- validator, err := s.getUpdator(current.Type)
- if err != nil {
- return err
- }
-
- err = validator.Validate(ctx, query.ParseQuery(current), entity)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func (s *Service) submitUpdate(ctx context.Context, current *query.Query, entity *resultprocessor.Update) error {
- err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
- id := entity.ID
-
- activeName, err := validation.GetFieldName(entity, entity.ActiveVersion)
- if err != nil {
- return err
- }
- onlyactive := validation.AreAllPointersNilExcept(entity, activeName)
-
- var latestVersion int32
- if !onlyactive {
- latestVersion, err = q.AddLatestQueryVersion(ctx, id)
- if err != nil {
- return err
- }
- }
-
- if entity.ActiveVersion != nil {
- err := q.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
- Queryid: id,
- Versionid: *entity.ActiveVersion,
- })
- if err != nil {
- return err
- }
- }
-
- if entity.RequiredQueryIDs != nil {
- addIDs := getSetDifference(entity.RequiredQueryIDs, current.RequiredQueryIDs)
- for _, qID := range addIDs {
- err := q.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
- Queryid: id,
- Requiredqueryid: qID,
- Addedversion: latestVersion,
- })
- if err != nil {
- return err
- }
- }
-
- removeIDs := getSetDifference(current.RequiredQueryIDs, entity.RequiredQueryIDs)
- for _, qID := range removeIDs {
- err := q.RemoveRequiredQuery(ctx, &repository.RemoveRequiredQueryParams{
- Queryid: id,
- Requiredqueryid: qID,
- Removedversion: &latestVersion,
- })
- if err != nil {
- return err
- }
- }
- }
-
- if entity.Config != nil && *entity.Config != "" {
- err = q.SetQueryConfig(ctx, &repository.SetQueryConfigParams{
- Queryid: id,
- Config: []byte(*entity.Config),
- Addedversion: latestVersion,
- })
- if err != nil {
- return err
- }
- }
-
- slog.Debug("query updated", "update", *entity)
-
- return nil
- })
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func getSetDifference(setA *[]uuid.UUID, setB *[]uuid.UUID) []uuid.UUID {
- if setA == nil {
- return []uuid.UUID{}
- } else if setB == nil {
- return *setA
- }
-
- diff := []uuid.UUID{}
- for _, q := range *setA {
- isFound := false
- for _, eq := range *setB {
- if q == eq {
- isFound = true
- break
- }
- }
- if !isFound {
- diff = append(diff, q)
- }
- }
-
- return diff
-}
-
-func (s *Service) getUpdator(qType resultprocessor.Type) (resultprocessor.Updator, error) {
- switch qType {
- case resultprocessor.TypeJsonExtractor:
- return jsonextractor.NewUpdator(), nil
- case resultprocessor.TypeContextFull:
- return contextfull.NewUpdator(), nil
- default:
- return nil, fmt.Errorf("attempting to process invalid query type")
- }
-}
diff --git a/internal/query/update/update_test.go b/internal/query/update/update_test.go
deleted file mode 100644
index 1d631c61..00000000
--- a/internal/query/update/update_test.go
+++ /dev/null
@@ -1,85 +0,0 @@
-package queryupdate_test
-
-import (
- "fmt"
- "testing"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/query"
- resultprocessor "queryorchestration/internal/query/result/processor"
- queryupdate "queryorchestration/internal/query/update"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/serviceconfig/queue/queryversionsync"
- queuemock "queryorchestration/mocks/queue"
-
- "github.com/stretchr/testify/require"
-
- "github.com/aws/aws-sdk-go-v2/service/sqs"
- "github.com/google/uuid"
- "github.com/jackc/pgx/v5"
- "github.com/pashagolub/pgxmock/v3"
- "github.com/stretchr/testify/mock"
-)
-
-type QueryUpdateConfig struct {
- serviceconfig.BaseConfig
- queryversionsync.QueryVersionSyncConfig
-}
-
-func TestUpdate(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- mockSQS := queuemock.NewMockSQSClient(t)
- cfg.QueueClient = mockSQS
- cfg.QueryVersionSyncURL = "/here"
- svc := queryupdate.New(cfg, &queryupdate.Services{
- Query: query.New(cfg),
- })
-
- config := "{\"path\":\"example_path\"}"
- existing := query.Query{
- ID: uuid.New(),
- ActiveVersion: int32(1),
- LatestVersion: int32(1),
- }
- av := int32(2)
- jcfg := `{"path":"id"}`
- update := &resultprocessor.Update{
- ID: existing.ID,
- ActiveVersion: &av,
- Config: &jcfg,
- }
-
- pool.ExpectQuery("name: GetQuery :one").WithArgs(update.ID).WillReturnRows(
- pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
- AddRow(existing.ID, repository.QuerytypeJsonExtractor, existing.ActiveVersion, existing.LatestVersion, []byte(config), []uuid.UUID{}),
- )
- pool.ExpectBeginTx(pgx.TxOptions{})
- pool.ExpectQuery("name: AddLatestQueryVersion :one").WithArgs(update.ID).WillReturnRows(
- pgxmock.NewRows([]string{"version"}).
- AddRow(int32(2)),
- )
- pool.ExpectExec("name: AddActiveQueryVersion :exec").WithArgs(update.ID, av).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectExec("name: SetQueryConfig :exec").WithArgs(update.ID, []byte(*update.Config), int32(2)).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectCommit()
-
- mockSQS.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.QueryVersionSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", existing.ID.String())
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
- err = svc.Update(ctx, update)
- require.NoError(t, err)
-}
diff --git a/internal/query/update/updateprivate_test.go b/internal/query/update/updateprivate_test.go
deleted file mode 100644
index b12ff78a..00000000
--- a/internal/query/update/updateprivate_test.go
+++ /dev/null
@@ -1,575 +0,0 @@
-package queryupdate
-
-import (
- "errors"
- "fmt"
- "testing"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/query"
- resultprocessor "queryorchestration/internal/query/result/processor"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/serviceconfig/queue/queryversionsync"
- queuemock "queryorchestration/mocks/queue"
-
- "github.com/stretchr/testify/require"
-
- "github.com/aws/aws-sdk-go-v2/service/sqs"
- "github.com/google/uuid"
- "github.com/jackc/pgx/v5"
- "github.com/pashagolub/pgxmock/v3"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
-)
-
-type QueryUpdateConfig struct {
- serviceconfig.BaseConfig
- queryversionsync.QueryVersionSyncConfig
-}
-
-func TestGetUpdator(t *testing.T) {
- cfg := &QueryUpdateConfig{}
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- queryType := resultprocessor.Type(resultprocessor.TypeContextFull)
- updator, err := svc.getUpdator(queryType)
- require.NoError(t, err)
- assert.NotNil(t, updator)
-
- queryType = resultprocessor.Type(resultprocessor.TypeJsonExtractor)
- updator, err = svc.getUpdator(queryType)
- require.NoError(t, err)
- assert.NotNil(t, updator)
-
- queryType = resultprocessor.Type(-1)
- _, err = svc.getUpdator(queryType)
- assert.Error(t, err)
-}
-
-func TestSubmitUpdate(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- config := "{\"path\":\"example_path\"}"
- q := query.Query{
- ID: uuid.New(),
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- },
- Config: &config,
- ActiveVersion: int32(1),
- LatestVersion: int32(2),
- }
- aV := int32(10)
- update := &resultprocessor.Update{
- ID: q.ID,
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- },
- Config: q.Config,
- ActiveVersion: &aV,
- }
-
- pool.ExpectBeginTx(pgx.TxOptions{})
- pool.ExpectQuery("name: AddLatestQueryVersion :one").WithArgs(q.ID).WillReturnRows(
- pgxmock.NewRows([]string{"version"}).
- AddRow(int32(3)),
- )
- pool.ExpectExec("name: AddActiveQueryVersion :exec").WithArgs(q.ID, aV).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectExec("name: AddRequiredQuery :exec").WithArgs(update.ID, (*update.RequiredQueryIDs)[0], pgxmock.AnyArg()).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectExec("name: RemoveRequiredQuery :exec").WithArgs(pgxmock.AnyArg(), (*q.RequiredQueryIDs)[0], update.ID).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectExec("name: SetQueryConfig :exec").WithArgs(update.ID, []byte(*update.Config), int32(3)).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectCommit()
-
- err = svc.submitUpdate(ctx, &q, update)
- require.NoError(t, err)
-}
-
-func TestSubmitUpdateRollback(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- q := query.Query{
- ID: uuid.New(),
- ActiveVersion: int32(1),
- LatestVersion: int32(2),
- }
- aV := int32(10)
- update := &resultprocessor.Update{
- ID: q.ID,
- ActiveVersion: &aV,
- }
-
- pool.ExpectBeginTx(pgx.TxOptions{})
- msg := "database failure"
- pool.ExpectExec("name: AddActiveQueryVersion :exec").WithArgs(q.ID, aV).
- WillReturnError(errors.New(msg))
- pool.ExpectCommit()
-
- err = svc.submitUpdate(ctx, &q, update)
- assert.EqualError(t, err, msg)
-}
-
-func TestSubmitUpdateRequiredQueries(t *testing.T) {
- t.Run("all fields", func(t *testing.T) {
-
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- config := "{\"path\":\"example_path\"}"
- q := query.Query{
- ID: uuid.New(),
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- uuid.New(),
- uuid.New(),
- },
- Config: &config,
- ActiveVersion: int32(1),
- LatestVersion: int32(2),
- }
- update := &resultprocessor.Update{
- ID: q.ID,
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- uuid.New(),
- (*q.RequiredQueryIDs)[0],
- },
- }
-
- pool.ExpectBeginTx(pgx.TxOptions{})
- pool.ExpectQuery("name: AddLatestQueryVersion :one").WithArgs(update.ID).WillReturnRows(
- pgxmock.NewRows([]string{"version"}).
- AddRow(int32(1)),
- )
- pool.ExpectExec("name: AddRequiredQuery :exec").WithArgs(update.ID, (*update.RequiredQueryIDs)[0], pgxmock.AnyArg()).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectExec("name: AddRequiredQuery :exec").WithArgs(update.ID, (*update.RequiredQueryIDs)[1], pgxmock.AnyArg()).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectExec("name: RemoveRequiredQuery :exec").WithArgs(pgxmock.AnyArg(), (*q.RequiredQueryIDs)[1], update.ID).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectExec("name: RemoveRequiredQuery :exec").WithArgs(pgxmock.AnyArg(), (*q.RequiredQueryIDs)[2], update.ID).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectCommit()
-
- err = svc.submitUpdate(ctx, &q, update)
- require.NoError(t, err)
- })
- t.Run("all nil", func(t *testing.T) {
-
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- config := "{\"path\":\"example_path\"}"
- q := query.Query{
- ID: uuid.New(),
- RequiredQueryIDs: &[]uuid.UUID{
- uuid.New(),
- uuid.New(),
- uuid.New(),
- },
- Config: &config,
- ActiveVersion: 1,
- LatestVersion: 2,
- }
- av := int32(2)
- update := &resultprocessor.Update{
- ID: q.ID,
- ActiveVersion: &av,
- }
-
- pool.ExpectBeginTx(pgx.TxOptions{})
- pool.ExpectExec("name: AddActiveQueryVersion :exec").WithArgs(q.ID, av).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectCommit()
-
- err = svc.submitUpdate(ctx, &q, update)
- require.NoError(t, err)
- })
-}
-
-func TestSubmitUpdateActiveVersion(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- q := query.Query{
- ID: uuid.New(),
- ActiveVersion: int32(1),
- LatestVersion: int32(2),
- }
- aV := int32(10)
- update := &resultprocessor.Update{
- ID: q.ID,
- ActiveVersion: &aV,
- }
-
- pool.ExpectBeginTx(pgx.TxOptions{})
- pool.ExpectExec("name: AddActiveQueryVersion :exec").WithArgs(q.ID, aV).
- WillReturnResult(pgxmock.NewResult("", 1))
- pool.ExpectCommit()
-
- err = svc.submitUpdate(ctx, &q, update)
- require.NoError(t, err)
-}
-
-func TestGetSetDifference(t *testing.T) {
- commonUUID := uuid.New()
- listA := []uuid.UUID{uuid.New(), commonUUID}
- listB := []uuid.UUID{uuid.New(), commonUUID}
-
- assert.ElementsMatch(t, []uuid.UUID{}, getSetDifference(nil, nil))
- assert.ElementsMatch(t, listA, getSetDifference(&listA, nil))
- assert.ElementsMatch(t, []uuid.UUID{}, getSetDifference(nil, &listB))
- assert.ElementsMatch(t, []uuid.UUID{listA[0]}, getSetDifference(&listA, &listB))
- assert.ElementsMatch(t, []uuid.UUID{listB[0]}, getSetDifference(&listB, &listA))
- assert.ElementsMatch(t, []uuid.UUID{}, getSetDifference(&listA, &listA))
-}
-
-func TestNormalizeUpdate(t *testing.T) {
- t.Run("all fields to change", func(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- current := &query.Query{
- ID: uuid.New(),
- ActiveVersion: int32(1),
- LatestVersion: int32(2),
- Type: resultprocessor.TypeJsonExtractor,
- }
- qcfg := "{}"
- aV := int32(2)
- update := &resultprocessor.Update{
- ID: current.ID,
- Config: &qcfg,
- ActiveVersion: &aV,
- RequiredQueryIDs: &[]uuid.UUID{},
- }
-
- err = svc.normalizeUpdate(ctx, current, update)
- require.NoError(t, err)
- assert.EqualExportedValues(t, resultprocessor.Update{
- ID: current.ID,
- ActiveVersion: &aV,
- Config: &qcfg,
- RequiredQueryIDs: nil,
- }, *update)
- })
-
- t.Run("no changes", func(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- current := &query.Query{
- ID: uuid.New(),
- ActiveVersion: int32(1),
- LatestVersion: int32(2),
- Type: resultprocessor.TypeJsonExtractor,
- }
- update := &resultprocessor.Update{
- ID: current.ID,
- }
- err = svc.normalizeUpdate(ctx, current, update)
- assert.EqualError(t, err, "no changes")
- })
-
- t.Run("same active version", func(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- version := int32(1)
- current := &query.Query{
- ID: uuid.New(),
- ActiveVersion: version,
- LatestVersion: int32(2),
- Type: resultprocessor.TypeJsonExtractor,
- }
- update := &resultprocessor.Update{
- ID: current.ID,
- ActiveVersion: &version,
- }
- err = svc.normalizeUpdate(ctx, current, update)
- assert.EqualError(t, err, "no changes")
- })
-
- t.Run("ONLY active version to next latest", func(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- current := &query.Query{
- ID: uuid.New(),
- ActiveVersion: 1,
- LatestVersion: 2,
- Type: resultprocessor.TypeJsonExtractor,
- }
- version := int32(3)
- update := &resultprocessor.Update{
- ID: current.ID,
- ActiveVersion: &version,
- }
- err = svc.normalizeUpdate(ctx, current, update)
- assert.EqualError(t, err, "no changes")
- })
-}
-
-func TestNormalizeUpdateRequiredQueryIDs(t *testing.T) {
- t.Run("Empty Array", func(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- current := &query.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- }
- update := &resultprocessor.Update{
- ID: current.ID,
- RequiredQueryIDs: &[]uuid.UUID{},
- }
-
- err = svc.normalizeUpdateRequiredQueryIDs(ctx, current, update)
- require.NoError(t, err)
- assert.EqualExportedValues(t, resultprocessor.Update{
- ID: current.ID,
- }, *update)
- })
-
- t.Run("entry doesn't exist", func(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- current := &query.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- }
- update := &resultprocessor.Update{
- ID: current.ID,
- RequiredQueryIDs: &[]uuid.UUID{uuid.New()},
- }
- dbids := *update.RequiredQueryIDs
-
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(false),
- )
-
- err = svc.normalizeUpdateRequiredQueryIDs(ctx, current, update)
- assert.Error(t, err)
- })
- t.Run("in dependency tree", func(t *testing.T) {
- ctx := t.Context()
-
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- current := &query.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- }
- update := &resultprocessor.Update{
- ID: current.ID,
- RequiredQueryIDs: &[]uuid.UUID{uuid.New()},
- }
-
- dbids := *update.RequiredQueryIDs
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(true),
- )
- pool.ExpectQuery("name: IsQueryInDependencyTree :one").WithArgs(dbids, current.ID).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(true),
- )
-
- err = svc.normalizeUpdateRequiredQueryIDs(ctx, current, update)
- assert.Error(t, err)
- })
- t.Run("same update and current", func(t *testing.T) {
- ctx := t.Context()
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryUpdateConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- svc := New(cfg, &Services{
- Query: query.New(cfg),
- })
-
- current := &query.Query{
- ID: uuid.New(),
- Type: resultprocessor.TypeJsonExtractor,
- }
- update := &resultprocessor.Update{
- ID: current.ID,
- }
- ids := []uuid.UUID{uuid.New()}
- update.RequiredQueryIDs = &ids
- current.RequiredQueryIDs = &ids
-
- dbids := *update.RequiredQueryIDs
- pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(true),
- )
- pool.ExpectQuery("name: IsQueryInDependencyTree :one").WithArgs(dbids, ¤t.ID).WillReturnRows(
- pgxmock.NewRows([]string{"all_exist"}).
- AddRow(false),
- )
-
- err = svc.normalizeUpdateRequiredQueryIDs(ctx, current, update)
- require.NoError(t, err)
- assert.EqualExportedValues(t, resultprocessor.Update{
- ID: current.ID,
- }, *update)
- })
-}
-
-func TestInformUpdate(t *testing.T) {
- t.Run("nil active version", func(t *testing.T) {
- ctx := t.Context()
-
- cfg := &QueryUpdateConfig{}
- mockSQS := queuemock.NewMockSQSClient(t)
- cfg.QueueClient = mockSQS
- cfg.QueryVersionSyncURL = "/here"
- svc := Service{
- cfg: cfg,
- }
-
- entity := &resultprocessor.Update{
- ID: uuid.New(),
- ActiveVersion: nil,
- }
-
- err := svc.informUpdate(ctx, entity)
- require.NoError(t, err)
- })
- t.Run("update active version", func(t *testing.T) {
- ctx := t.Context()
-
- cfg := &QueryUpdateConfig{}
- mockSQS := queuemock.NewMockSQSClient(t)
- cfg.QueueClient = mockSQS
- cfg.QueryVersionSyncURL = "/here"
- svc := Service{
- cfg: cfg,
- }
-
- av := int32(2)
- entity := &resultprocessor.Update{
- ID: uuid.New(),
- ActiveVersion: &av,
- }
-
- mockSQS.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.QueryVersionSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", entity.ID.String())
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
- err := svc.informUpdate(ctx, entity)
- require.NoError(t, err)
- })
-}
diff --git a/internal/query/versionsync/service.go b/internal/query/versionsync/service.go
deleted file mode 100644
index c5905be0..00000000
--- a/internal/query/versionsync/service.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package queryversionsync
-
-import (
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/serviceconfig/queue/clientsync"
-)
-
-type ConfigProvider interface {
- serviceconfig.ConfigProvider
- clientsync.ConfigProvider
-}
-
-type Service struct {
- cfg ConfigProvider
-}
-
-func New(cfg ConfigProvider) *Service {
- return &Service{
- cfg,
- }
-}
diff --git a/internal/query/versionsync/sync.go b/internal/query/versionsync/sync.go
deleted file mode 100644
index c072ed21..00000000
--- a/internal/query/versionsync/sync.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package queryversionsync
-
-import (
- "context"
- "database/sql"
- "errors"
-
- clientsyncrunner "queryorchestration/api/clientSyncRunner"
- "queryorchestration/internal/serviceconfig/queue"
-
- "github.com/google/uuid"
-)
-
-func (s *Service) Sync(ctx context.Context, queryId uuid.UUID) error {
- clientIds, err := s.cfg.GetDBQueries().ListQueryClientIDs(ctx, &queryId)
- if err != nil && errors.Is(err, sql.ErrNoRows) {
- return nil
- } else if err != nil {
- return err
- }
-
- for _, id := range clientIds {
- err := s.cfg.SendToQueue(ctx, &queue.SendParams{
- QueueURL: s.cfg.GetClientSyncURL(),
- Body: clientsyncrunner.Body{
- ClientID: id,
- },
- })
- if err != nil {
- return err
- }
- }
-
- return nil
-}
diff --git a/internal/query/versionsync/sync_test.go b/internal/query/versionsync/sync_test.go
deleted file mode 100644
index ff0e0ce4..00000000
--- a/internal/query/versionsync/sync_test.go
+++ /dev/null
@@ -1,72 +0,0 @@
-package queryversionsync
-
-import (
- "fmt"
- "testing"
-
- "queryorchestration/internal/database/repository"
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/serviceconfig/queue/clientsync"
- queuemock "queryorchestration/mocks/queue"
-
- "github.com/stretchr/testify/require"
-
- "github.com/aws/aws-sdk-go-v2/service/sqs"
- "github.com/google/uuid"
- "github.com/pashagolub/pgxmock/v3"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
-)
-
-type QueryVersionSyncConfig struct {
- serviceconfig.BaseConfig
- clientsync.ClientSyncConfig
-}
-
-func TestSync(t *testing.T) {
- ctx := t.Context()
- pool, err := pgxmock.NewPool()
- require.NoError(t, err)
- cfg := &QueryVersionSyncConfig{}
- cfg.DBPool = pool
- cfg.DBQueries = repository.New(pool)
- mockSQS := queuemock.NewMockSQSClient(t)
- cfg.QueueClient = mockSQS
- cfg.ClientSyncURL = "/i/am/here"
-
- svc := Service{cfg}
-
- queryId := uuid.New()
- clientIds := []string{
- "hello",
- "aarete",
- }
-
- pool.ExpectQuery("name: ListQueryClientIDs :many").WithArgs(&queryId).
- WillReturnRows(
- pgxmock.NewRows([]string{"clientId"}).
- AddRow(clientIds[0]).
- AddRow(clientIds[1]),
- )
-
- mockSQS.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", clientIds[0])
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
- mockSQS.EXPECT().
- SendMessage(
- mock.Anything,
- mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
- return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", clientIds[1])
- }),
- mock.Anything,
- ).
- Return(&sqs.SendMessageOutput{}, nil)
-
- assert.Nil(t, svc.Sync(ctx, queryId))
-}
diff --git a/internal/server/runner/poll_test.go b/internal/server/runner/poll_test.go
index 12857fc5..8d488b8c 100644
--- a/internal/server/runner/poll_test.go
+++ b/internal/server/runner/poll_test.go
@@ -29,7 +29,7 @@ func TestPollMessages(t *testing.T) {
acfg := test.CreateAWSContainer(t, cfg)
test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
- cfg.QueueURL = test.CreateQueue(t, cfg, test.QueryRunnerName)
+ cfg.QueueURL = test.CreateQueue(t, cfg, test.ClientSyncRunnerName)
mockController := runnermock.NewMockController[MockStruct](t)
cfg.Controller = mockController
@@ -54,7 +54,7 @@ func TestPollMessage(t *testing.T) {
acfg := test.CreateAWSContainer(t, cfg)
test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
- cfg.QueueURL = test.CreateQueue(t, cfg, test.QueryRunnerName)
+ cfg.QueueURL = test.CreateQueue(t, cfg, test.ClientSyncRunnerName)
mockController := runnermock.NewMockController[MockStruct](t)
cfg.Controller = mockController
@@ -94,7 +94,7 @@ func TestProcessMessage(t *testing.T) {
acfg := test.CreateAWSContainer(t, cfg)
test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
- cfg.QueueURL = test.CreateQueue(t, cfg, test.QueryRunnerName)
+ cfg.QueueURL = test.CreateQueue(t, cfg, test.ClientSyncRunnerName)
mockController := runnermock.NewMockController[MockStruct](t)
cfg.Controller = mockController
@@ -136,7 +136,7 @@ func TestProcessMessage(t *testing.T) {
acfg := test.CreateAWSContainer(t, cfg)
test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
- cfg.QueueURL = test.CreateQueue(t, cfg, test.QueryRunnerName)
+ cfg.QueueURL = test.CreateQueue(t, cfg, test.ClientSyncRunnerName)
mockController := runnermock.NewMockController[MockStruct](t)
cfg.Controller = mockController
diff --git a/internal/serviceconfig/database/pool.go b/internal/serviceconfig/database/pool.go
index 53110c92..b1d54c40 100644
--- a/internal/serviceconfig/database/pool.go
+++ b/internal/serviceconfig/database/pool.go
@@ -32,9 +32,9 @@ func (b *DBConfig) SetDBPoolConfig() error {
return err
}
- b.DBEnumTypes = []string{
- "querytype",
- }
+ // Note: querytype enum was removed as part of query functionality removal.
+ // If new custom database enum types are added, register them here.
+ b.DBEnumTypes = []string{}
config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
for _, enumName := range b.DBEnumTypes {
diff --git a/internal/serviceconfig/queue/config_test.go b/internal/serviceconfig/queue/config_test.go
index 721897fa..6f44d355 100644
--- a/internal/serviceconfig/queue/config_test.go
+++ b/internal/serviceconfig/queue/config_test.go
@@ -32,9 +32,9 @@ func TestPingQueueByURL(t *testing.T) {
acfg := test.CreateAWSContainer(t, cfg)
test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
- cfg.QueryURL = test.CreateQueue(t, cfg, test.QueryRunnerName)
+ cfg.ClientSyncURL = test.CreateQueue(t, cfg, test.ClientSyncRunnerName)
- err := cfg.PingQueueByURL(t.Context(), cfg.QueryURL)
+ err := cfg.PingQueueByURL(t.Context(), cfg.ClientSyncURL)
require.NoError(t, err)
}
diff --git a/internal/serviceconfig/queue/delete_test.go b/internal/serviceconfig/queue/delete_test.go
index ad917c08..2c146057 100644
--- a/internal/serviceconfig/queue/delete_test.go
+++ b/internal/serviceconfig/queue/delete_test.go
@@ -15,28 +15,28 @@ func TestDelete(t *testing.T) {
acfg := test.CreateAWSContainer(t, cfg)
test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
- cfg.QueryURL = test.CreateQueue(t, cfg, test.QueryRunnerName)
+ cfg.ClientSyncURL = test.CreateQueue(t, cfg, test.ClientSyncRunnerName)
err := cfg.SendToQueue(t.Context(), &queue.SendParams{
- QueueURL: cfg.GetQueryURL(),
+ QueueURL: cfg.GetClientSyncURL(),
Body: struct{}{},
})
require.NoError(t, err)
msg := test.AssertMessage(t, cfg, &queue.ReceiveParams{
- QueueURL: cfg.GetQueryURL(),
+ QueueURL: cfg.GetClientSyncURL(),
})
params := &queue.DeleteParams{
ReceiptHandle: msg.ReceiptHandle,
- QueueURL: cfg.GetQueryURL(),
+ QueueURL: cfg.GetClientSyncURL(),
}
err = cfg.DeleteFromQueue(t.Context(), params)
require.NoError(t, err)
ares, err := cfg.ReceiveFromQueue(t.Context(), &queue.ReceiveParams{
- QueueURL: cfg.GetQueryURL(),
+ QueueURL: cfg.GetClientSyncURL(),
})
require.NoError(t, err)
require.Len(t, ares.Messages, 0)
diff --git a/internal/serviceconfig/queue/query/config.go b/internal/serviceconfig/queue/query/config.go
deleted file mode 100644
index 0744f2dc..00000000
--- a/internal/serviceconfig/queue/query/config.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package query
-
-const EnvName = "QUERY_URL"
-
-type QueryConfig struct {
- QueryURL string `env:"QUERY_URL,required,notEmpty"`
-}
-
-func (c *QueryConfig) GetQueryURL() string {
- return c.QueryURL
-}
-
-type ConfigProvider interface {
- GetQueryURL() string
-}
diff --git a/internal/serviceconfig/queue/query/config_test.go b/internal/serviceconfig/queue/query/config_test.go
deleted file mode 100644
index 3725a71c..00000000
--- a/internal/serviceconfig/queue/query/config_test.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package query_test
-
-import (
- "testing"
-
- "queryorchestration/internal/serviceconfig/queue/query"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestGetQueryURL(t *testing.T) {
- cfg := query.QueryConfig{}
-
- name := cfg.GetQueryURL()
- assert.Equal(t, "", name)
-
- cfg.QueryURL = "name"
- name = cfg.GetQueryURL()
- assert.Equal(t, "name", name)
- assert.Equal(t, cfg.QueryURL, name)
-}
diff --git a/internal/serviceconfig/queue/querysync/config.go b/internal/serviceconfig/queue/querysync/config.go
deleted file mode 100644
index ca17c898..00000000
--- a/internal/serviceconfig/queue/querysync/config.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package querysync
-
-const EnvName = "QUERY_SYNC_URL"
-
-type QuerySyncConfig struct {
- QuerySyncURL string `env:"QUERY_SYNC_URL,required,notEmpty"`
-}
-
-func (c *QuerySyncConfig) GetQuerySyncURL() string {
- return c.QuerySyncURL
-}
-
-type ConfigProvider interface {
- GetQuerySyncURL() string
-}
diff --git a/internal/serviceconfig/queue/querysync/config_test.go b/internal/serviceconfig/queue/querysync/config_test.go
deleted file mode 100644
index e2643095..00000000
--- a/internal/serviceconfig/queue/querysync/config_test.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package querysync_test
-
-import (
- "testing"
-
- "queryorchestration/internal/serviceconfig/queue/querysync"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestGetQuerySyncURL(t *testing.T) {
- cfg := querysync.QuerySyncConfig{}
-
- name := cfg.GetQuerySyncURL()
- assert.Equal(t, "", name)
-
- cfg.QuerySyncURL = "name"
- name = cfg.GetQuerySyncURL()
- assert.Equal(t, "name", name)
- assert.Equal(t, cfg.QuerySyncURL, name)
-}
diff --git a/internal/serviceconfig/queue/queryversionsync/config.go b/internal/serviceconfig/queue/queryversionsync/config.go
deleted file mode 100644
index 88db2da4..00000000
--- a/internal/serviceconfig/queue/queryversionsync/config.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package queryversionsync
-
-const EnvName = "QUERY_VERSION_SYNC_URL"
-
-type QueryVersionSyncConfig struct {
- QueryVersionSyncURL string `env:"QUERY_VERSION_SYNC_URL,required,notEmpty"`
-}
-
-func (c *QueryVersionSyncConfig) GetQueryVersionSyncURL() string {
- return c.QueryVersionSyncURL
-}
-
-type ConfigProvider interface {
- GetQueryVersionSyncURL() string
-}
diff --git a/internal/serviceconfig/queue/queryversionsync/config_test.go b/internal/serviceconfig/queue/queryversionsync/config_test.go
deleted file mode 100644
index 32f4bf47..00000000
--- a/internal/serviceconfig/queue/queryversionsync/config_test.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package queryversionsync_test
-
-import (
- "testing"
-
- "queryorchestration/internal/serviceconfig/queue/queryversionsync"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestGetQueryVersionSyncURL(t *testing.T) {
- cfg := queryversionsync.QueryVersionSyncConfig{}
-
- name := cfg.GetQueryVersionSyncURL()
- assert.Equal(t, "", name)
-
- cfg.QueryVersionSyncURL = "name"
- name = cfg.GetQueryVersionSyncURL()
- assert.Equal(t, "name", name)
- assert.Equal(t, cfg.QueryVersionSyncURL, name)
-}
diff --git a/internal/serviceconfig/queue/receive_test.go b/internal/serviceconfig/queue/receive_test.go
index b0adc98b..4799bd42 100644
--- a/internal/serviceconfig/queue/receive_test.go
+++ b/internal/serviceconfig/queue/receive_test.go
@@ -16,10 +16,10 @@ func TestReceive(t *testing.T) {
acfg := test.CreateAWSContainer(t, cfg)
test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
- cfg.QueryURL = test.CreateQueue(t, cfg, test.QueryRunnerName)
+ cfg.ClientSyncURL = test.CreateQueue(t, cfg, test.ClientSyncRunnerName)
err := cfg.SendToQueue(t.Context(), &queue.SendParams{
- QueueURL: cfg.GetQueryURL(),
+ QueueURL: cfg.GetClientSyncURL(),
Body: struct {
Field string
}{
@@ -29,7 +29,7 @@ func TestReceive(t *testing.T) {
require.NoError(t, err)
params := &queue.ReceiveParams{
- QueueURL: cfg.GetQueryURL(),
+ QueueURL: cfg.GetClientSyncURL(),
}
ares, err := cfg.ReceiveFromQueue(t.Context(), params)
diff --git a/internal/serviceconfig/queue/send_test.go b/internal/serviceconfig/queue/send_test.go
index 7a61ab24..8e82b33d 100644
--- a/internal/serviceconfig/queue/send_test.go
+++ b/internal/serviceconfig/queue/send_test.go
@@ -7,17 +7,19 @@ import (
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue"
- "queryorchestration/internal/serviceconfig/queue/query"
+ "queryorchestration/internal/serviceconfig/queue/clientsync"
"queryorchestration/internal/test"
"github.com/stretchr/testify/require"
)
+// AWSConfig provides test configuration for queue operations.
+// Note: Query-related configuration has been removed. See remove_query_plan.md.
type AWSConfig struct {
serviceconfig.BaseConfig
queue.QueueConfig
objectstore.ObjectStoreConfig
- query.QueryConfig
+ clientsync.ClientSyncConfig
}
func TestSend(t *testing.T) {
@@ -25,15 +27,15 @@ func TestSend(t *testing.T) {
acfg := test.CreateAWSContainer(t, cfg)
test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
- cfg.QueryURL = test.CreateQueue(t, cfg, test.QueryRunnerName)
+ cfg.ClientSyncURL = test.CreateQueue(t, cfg, test.ClientSyncRunnerName)
params := &queue.SendParams{
- QueueURL: cfg.GetQueryURL(),
+ QueueURL: cfg.GetClientSyncURL(),
Body: struct{}{},
}
err := cfg.SendToQueue(t.Context(), params)
require.NoError(t, err)
- test.AssertMessageBody(t, cfg, cfg.GetQueryURL(), regexp.MustCompile("{}"))
+ test.AssertMessageBody(t, cfg, cfg.GetClientSyncURL(), regexp.MustCompile("{}"))
}
diff --git a/internal/test/api.go b/internal/test/api.go
index acc1e1a6..1950fa85 100644
--- a/internal/test/api.go
+++ b/internal/test/api.go
@@ -53,11 +53,11 @@ func CreateAPI(t testing.TB, ctx context.Context, cfg ContainerConfigProvider, n
}, cleanup
}
+// QueryAPI configuration. Query processing has been removed - only client sync remains.
var QueryAPI = API{
Name: QueryAPIName,
DownstreamQueues: []RunnerName{
ClientSyncRunnerName,
- QueryVersionSyncRunnerName,
},
}
diff --git a/internal/test/config.go b/internal/test/config.go
index 479f1baa..4c1e7dda 100644
--- a/internal/test/config.go
+++ b/internal/test/config.go
@@ -121,9 +121,10 @@ func WaitForAPIRoutes(uri string, timeout time.Duration) error {
client := &http.Client{Timeout: 2 * time.Second}
deadline := time.Now().Add(timeout)
- // Check /query endpoint - should return 200 with empty array when routes are ready
+ // Check /clients endpoint - should return a response when routes are ready
// A "404 page not found" means routes aren't registered yet
- routeURL := fmt.Sprintf("%s/query", uri)
+ // Note: /query endpoint was removed as part of query functionality removal
+ routeURL := fmt.Sprintf("%s/clients", uri)
slog.Info("Waiting for API routes to be registered", "url", routeURL, "timeout", timeout)
var lastErr error
diff --git a/internal/test/config_test.go b/internal/test/config_test.go
index aed76f3c..c0b2d3a3 100644
--- a/internal/test/config_test.go
+++ b/internal/test/config_test.go
@@ -152,7 +152,7 @@ func TestWaitForExternalReadiness(t *testing.T) {
func TestWaitForAPIRoutes(t *testing.T) {
t.Run("success when routes return 200", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.URL.Path == "/query" {
+ if r.URL.Path == "/clients" {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("[]"))
return
@@ -168,7 +168,7 @@ func TestWaitForAPIRoutes(t *testing.T) {
t.Run("success after routes become available", func(t *testing.T) {
attempts := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.URL.Path == "/query" {
+ if r.URL.Path == "/clients" {
attempts++
if attempts < 3 {
// Simulate "404 page not found" before routes are registered
@@ -192,7 +192,7 @@ func TestWaitForAPIRoutes(t *testing.T) {
t.Run("success when routes return 401", func(t *testing.T) {
// 401 means the route exists but requires auth - route is registered
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.URL.Path == "/query" {
+ if r.URL.Path == "/clients" {
w.WriteHeader(http.StatusUnauthorized)
return
}
diff --git a/internal/test/queryAPI/service.go b/internal/test/queryAPI/service.go
index a78f60ad..0a397c87 100644
--- a/internal/test/queryAPI/service.go
+++ b/internal/test/queryAPI/service.go
@@ -14,12 +14,13 @@ import (
queryapi "queryorchestration/pkg/queryAPI"
- "github.com/google/uuid"
- "github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
+// Note: Query-related helper functions have been removed.
+// See remove_query_plan.md for details.
+
type File struct {
ClientID queryapi.ClientID
Filename string
@@ -69,22 +70,6 @@ func CreateClientWithSync(t testing.TB, client queryapi.ClientWithResponsesInter
return clientCreateRes.JSON201.Id
}
-func SetQueryForClient(t testing.TB, client queryapi.ClientWithResponsesInterface, clientId queryapi.ClientID, queryId uuid.UUID) {
- t.Helper()
-
- newActiveVersion := int32(1)
- _, err := client.SetCollectorByClientIdWithResponse(t.Context(), clientId, queryapi.CollectorSet{
- ActiveVersion: &newActiveVersion,
- Fields: &[]queryapi.CollectorField{
- {
- Name: "JSON_QUERY",
- QueryId: queryId,
- },
- },
- })
- require.NoError(t, err)
-}
-
func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.ClientWithResponsesInterface, id string, status queryapi.ClientStatus) {
t.Helper()
@@ -125,21 +110,3 @@ func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.Cli
}
}
}
-
-func CreateDependentQueries(t testing.TB, client *queryapi.ClientWithResponses) (uuid.UUID, uuid.UUID) {
- contextQueryRes, err := client.CreateQueryWithResponse(t.Context(), queryapi.QueryCreate{
- Type: queryapi.CONTEXTFULL,
- })
- require.NoError(t, err)
- test.AssertStatus(t, http.StatusCreated, contextQueryRes.HTTPResponse)
- jcfg := `{"path":"keyone"}`
- jsonQueryRes, err := client.CreateQueryWithResponse(t.Context(), queryapi.QueryCreate{
- Type: queryapi.JSONEXTRACTOR,
- Config: &jcfg,
- RequiredQueries: &[]types.UUID{contextQueryRes.JSON201.Id},
- })
- require.NoError(t, err)
- test.AssertStatus(t, http.StatusCreated, jsonQueryRes.HTTPResponse)
-
- return contextQueryRes.JSON201.Id, jsonQueryRes.JSON201.Id
-}
diff --git a/internal/test/queue.go b/internal/test/queue.go
index 10ccec9e..214a03a1 100644
--- a/internal/test/queue.go
+++ b/internal/test/queue.go
@@ -3,7 +3,6 @@ package test
import (
"context"
"fmt"
- "log"
"log/slog"
"regexp"
"testing"
@@ -112,55 +111,3 @@ func AssertMessageAttr(t testing.TB, cfg serviceconfig.ConfigProvider, url strin
assert.NotNil(t, message.MessageAttributes[name])
assert.Regexp(t, value, *(message.MessageAttributes[name]).StringValue)
}
-
-func AssertMessageBodies(t testing.TB, cfg serviceconfig.ConfigProvider, url string, bodies []*regexp.Regexp) {
- t.Helper()
- timeout := time.After(30 * time.Second)
- tick := time.NewTicker(2 * time.Second)
- defer tick.Stop()
-
- for {
- select {
- case <-timeout:
- t.Fatal("assert timeout")
- case <-tick.C:
- slog.Info("receiving from queue")
-
- result, err := cfg.ReceiveFromQueue(t.Context(), &queue.ReceiveParams{
- QueueURL: url,
- VisibilityTimeout: 1,
- WaitTimeSeconds: 1,
- })
- if err != nil {
- continue
- } else if len(result.Messages) < 1 {
- continue
- }
-
- found := true
- for _, m := range result.Messages {
- foundBody := false
- for _, b := range bodies {
- log.Print(*m.Body)
- if b.MatchString(*m.Body) {
- foundBody = true
- break
- }
- }
-
- if !foundBody {
- found = false
- break
- }
- }
-
- if !found {
- continue
- }
-
- return
- case <-t.Context().Done():
- t.Fatal(t.Context().Err())
- }
- }
-}
diff --git a/internal/test/runner.go b/internal/test/runner.go
index 0ed8dbc0..bcabf310 100644
--- a/internal/test/runner.go
+++ b/internal/test/runner.go
@@ -9,38 +9,29 @@ import (
"queryorchestration/internal/serviceconfig/queue/documentinit"
"queryorchestration/internal/serviceconfig/queue/documentsync"
"queryorchestration/internal/serviceconfig/queue/documenttext"
- "queryorchestration/internal/serviceconfig/queue/query"
- "queryorchestration/internal/serviceconfig/queue/querysync"
- "queryorchestration/internal/serviceconfig/queue/queryversionsync"
"queryorchestration/internal/serviceconfig/queue/storeevent"
)
type RunnerName string
const (
- StoreEventRunnerName RunnerName = "storeEventRunner"
- DocInitRunnerName RunnerName = "docInitRunner"
- DocSyncRunnerName RunnerName = "docSyncRunner"
- DocCleanRunnerName RunnerName = "docCleanRunner"
- DocTextRunnerName RunnerName = "docTextRunner"
- QuerySyncRunnerName RunnerName = "querySyncRunner"
- QueryRunnerName RunnerName = "queryRunner"
- ClientSyncRunnerName RunnerName = "clientSyncRunner"
- QueryVersionSyncRunnerName RunnerName = "queryVersionSyncRunner"
+ StoreEventRunnerName RunnerName = "storeEventRunner"
+ DocInitRunnerName RunnerName = "docInitRunner"
+ DocSyncRunnerName RunnerName = "docSyncRunner"
+ DocCleanRunnerName RunnerName = "docCleanRunner"
+ DocTextRunnerName RunnerName = "docTextRunner"
+ ClientSyncRunnerName RunnerName = "clientSyncRunner"
)
type RunnerEnv string
const (
- StoreEventRunnerEnv RunnerEnv = storeevent.EnvName
- DocInitRunnerEnv RunnerEnv = documentinit.EnvName
- DocSyncRunnerEnv RunnerEnv = documentsync.EnvName
- DocCleanRunnerEnv RunnerEnv = documentclean.EnvName
- DocTextRunnerEnv RunnerEnv = documenttext.EnvName
- QuerySyncRunnerEnv RunnerEnv = querysync.EnvName
- QueryRunnerEnv RunnerEnv = query.EnvName
- ClientSyncRunnerEnv RunnerEnv = clientsync.EnvName
- QueryVersionSyncRunnerEnv RunnerEnv = queryversionsync.EnvName
+ StoreEventRunnerEnv RunnerEnv = storeevent.EnvName
+ DocInitRunnerEnv RunnerEnv = documentinit.EnvName
+ DocSyncRunnerEnv RunnerEnv = documentsync.EnvName
+ DocCleanRunnerEnv RunnerEnv = documentclean.EnvName
+ DocTextRunnerEnv RunnerEnv = documenttext.EnvName
+ ClientSyncRunnerEnv RunnerEnv = clientsync.EnvName
)
type Runner struct {
@@ -60,14 +51,8 @@ func GetRunnerEnvFromName(name RunnerName) RunnerEnv {
return DocCleanRunnerEnv
case DocTextRunnerName:
return DocTextRunnerEnv
- case QuerySyncRunnerName:
- return QuerySyncRunnerEnv
- case QueryRunnerName:
- return QueryRunnerEnv
case ClientSyncRunnerName:
return ClientSyncRunnerEnv
- case QueryVersionSyncRunnerName:
- return QueryVersionSyncRunnerEnv
}
return DocInitRunnerEnv
}
@@ -117,19 +102,11 @@ var DocCleanRunner = Runner{
DownstreamQueues: []RunnerName{DocTextRunnerName},
}
+// DocTextRunner is the terminal runner in the main document processing pipeline.
+// Query processing has been removed - pipeline ends after text extraction.
var DocTextRunner = Runner{
Name: DocTextRunnerName,
- DownstreamQueues: []RunnerName{QuerySyncRunnerName},
-}
-
-var QuerySyncRunner = Runner{
- Name: QuerySyncRunnerName,
- DownstreamQueues: []RunnerName{QueryRunnerName},
-}
-
-var QueryRunner = Runner{
- Name: QueryRunnerName,
- DownstreamQueues: []RunnerName{QueryRunnerName},
+ DownstreamQueues: []RunnerName{},
}
var ClientSyncRunner = Runner{
@@ -137,19 +114,11 @@ var ClientSyncRunner = Runner{
DownstreamQueues: []RunnerName{DocSyncRunnerName},
}
-var QueryVersionSyncRunner = Runner{
- Name: QueryVersionSyncRunnerName,
- DownstreamQueues: []RunnerName{ClientSyncRunnerName},
-}
-
var runners = []Runner{
StoreEventRunner,
DocInitRunner,
DocSyncRunner,
DocCleanRunner,
DocTextRunner,
- QuerySyncRunner,
- QueryRunner,
ClientSyncRunner,
- QueryVersionSyncRunner,
}
diff --git a/internal/test/runner_test.go b/internal/test/runner_test.go
index 90b8cb90..234d28f1 100644
--- a/internal/test/runner_test.go
+++ b/internal/test/runner_test.go
@@ -6,15 +6,13 @@ import (
"github.com/stretchr/testify/assert"
)
+// TestGetRunnerEnvFromName tests the runner environment name lookup.
+// Note: Query runner references have been removed. See remove_query_plan.md for details.
func TestGetRunnerEnvFromName(t *testing.T) {
assert.Equal(t, StoreEventRunnerEnv, GetRunnerEnvFromName(StoreEventRunnerName))
assert.Equal(t, DocInitRunnerEnv, GetRunnerEnvFromName(DocInitRunnerName))
assert.Equal(t, DocSyncRunnerEnv, GetRunnerEnvFromName(DocSyncRunnerName))
assert.Equal(t, DocCleanRunnerEnv, GetRunnerEnvFromName(DocCleanRunnerName))
assert.Equal(t, DocTextRunnerEnv, GetRunnerEnvFromName(DocTextRunnerName))
- assert.Equal(t, QuerySyncRunnerEnv, GetRunnerEnvFromName(QuerySyncRunnerName))
- assert.Equal(t, QueryRunnerEnv, GetRunnerEnvFromName(QueryRunnerName))
assert.Equal(t, ClientSyncRunnerEnv, GetRunnerEnvFromName(ClientSyncRunnerName))
- assert.Equal(t, QueryVersionSyncRunnerEnv, GetRunnerEnvFromName(QueryVersionSyncRunnerName))
- assert.Equal(t, DocInitRunnerEnv, GetRunnerEnvFromName(DocInitRunnerName))
}
diff --git a/mocks/queryapi/mock_ClientInterface.go b/mocks/queryapi/mock_ClientInterface.go
index 8d7608cb..d716229f 100644
--- a/mocks/queryapi/mock_ClientInterface.go
+++ b/mocks/queryapi/mock_ClientInterface.go
@@ -924,155 +924,6 @@ func (_c *MockClientInterface_CreateFolderWithBody_Call) RunAndReturn(run func(c
return _c
}
-// CreateQuery provides a mock function with given fields: ctx, body, reqEditors
-func (_m *MockClientInterface) CreateQuery(ctx context.Context, body queryapi.CreateQueryJSONRequestBody, reqEditors ...queryapi.RequestEditorFn) (*http.Response, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, body)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for CreateQuery")
- }
-
- var r0 *http.Response
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.CreateQueryJSONRequestBody, ...queryapi.RequestEditorFn) (*http.Response, error)); ok {
- return rf(ctx, body, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.CreateQueryJSONRequestBody, ...queryapi.RequestEditorFn) *http.Response); ok {
- r0 = rf(ctx, body, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*http.Response)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, queryapi.CreateQueryJSONRequestBody, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, body, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientInterface_CreateQuery_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateQuery'
-type MockClientInterface_CreateQuery_Call struct {
- *mock.Call
-}
-
-// CreateQuery is a helper method to define mock.On call
-// - ctx context.Context
-// - body queryapi.CreateQueryJSONRequestBody
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientInterface_Expecter) CreateQuery(ctx interface{}, body interface{}, reqEditors ...interface{}) *MockClientInterface_CreateQuery_Call {
- return &MockClientInterface_CreateQuery_Call{Call: _e.mock.On("CreateQuery",
- append([]interface{}{ctx, body}, reqEditors...)...)}
-}
-
-func (_c *MockClientInterface_CreateQuery_Call) Run(run func(ctx context.Context, body queryapi.CreateQueryJSONRequestBody, reqEditors ...queryapi.RequestEditorFn)) *MockClientInterface_CreateQuery_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-2)
- for i, a := range args[2:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(queryapi.CreateQueryJSONRequestBody), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientInterface_CreateQuery_Call) Return(_a0 *http.Response, _a1 error) *MockClientInterface_CreateQuery_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientInterface_CreateQuery_Call) RunAndReturn(run func(context.Context, queryapi.CreateQueryJSONRequestBody, ...queryapi.RequestEditorFn) (*http.Response, error)) *MockClientInterface_CreateQuery_Call {
- _c.Call.Return(run)
- return _c
-}
-
-// CreateQueryWithBody provides a mock function with given fields: ctx, contentType, body, reqEditors
-func (_m *MockClientInterface) CreateQueryWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn) (*http.Response, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, contentType, body)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for CreateQueryWithBody")
- }
-
- var r0 *http.Response
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, string, io.Reader, ...queryapi.RequestEditorFn) (*http.Response, error)); ok {
- return rf(ctx, contentType, body, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, string, io.Reader, ...queryapi.RequestEditorFn) *http.Response); ok {
- r0 = rf(ctx, contentType, body, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*http.Response)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, string, io.Reader, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, contentType, body, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientInterface_CreateQueryWithBody_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateQueryWithBody'
-type MockClientInterface_CreateQueryWithBody_Call struct {
- *mock.Call
-}
-
-// CreateQueryWithBody is a helper method to define mock.On call
-// - ctx context.Context
-// - contentType string
-// - body io.Reader
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientInterface_Expecter) CreateQueryWithBody(ctx interface{}, contentType interface{}, body interface{}, reqEditors ...interface{}) *MockClientInterface_CreateQueryWithBody_Call {
- return &MockClientInterface_CreateQueryWithBody_Call{Call: _e.mock.On("CreateQueryWithBody",
- append([]interface{}{ctx, contentType, body}, reqEditors...)...)}
-}
-
-func (_c *MockClientInterface_CreateQueryWithBody_Call) Run(run func(ctx context.Context, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn)) *MockClientInterface_CreateQueryWithBody_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-3)
- for i, a := range args[3:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(string), args[2].(io.Reader), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientInterface_CreateQueryWithBody_Call) Return(_a0 *http.Response, _a1 error) *MockClientInterface_CreateQueryWithBody_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientInterface_CreateQueryWithBody_Call) RunAndReturn(run func(context.Context, string, io.Reader, ...queryapi.RequestEditorFn) (*http.Response, error)) *MockClientInterface_CreateQueryWithBody_Call {
- _c.Call.Return(run)
- return _c
-}
-
// DeleteAdminUser provides a mock function with given fields: ctx, email, params, reqEditors
func (_m *MockClientInterface) DeleteAdminUser(ctx context.Context, email queryapi.UserEmail, params *queryapi.DeleteAdminUserParams, reqEditors ...queryapi.RequestEditorFn) (*http.Response, error) {
_va := make([]interface{}, len(reqEditors))
@@ -2408,80 +2259,6 @@ func (_c *MockClientInterface_GetIdentity_Call) RunAndReturn(run func(context.Co
return _c
}
-// GetQuery provides a mock function with given fields: ctx, id, reqEditors
-func (_m *MockClientInterface) GetQuery(ctx context.Context, id queryapi.QueryID, reqEditors ...queryapi.RequestEditorFn) (*http.Response, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, id)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for GetQuery")
- }
-
- var r0 *http.Response
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, ...queryapi.RequestEditorFn) (*http.Response, error)); ok {
- return rf(ctx, id, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, ...queryapi.RequestEditorFn) *http.Response); ok {
- r0 = rf(ctx, id, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*http.Response)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, queryapi.QueryID, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, id, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientInterface_GetQuery_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetQuery'
-type MockClientInterface_GetQuery_Call struct {
- *mock.Call
-}
-
-// GetQuery is a helper method to define mock.On call
-// - ctx context.Context
-// - id queryapi.QueryID
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientInterface_Expecter) GetQuery(ctx interface{}, id interface{}, reqEditors ...interface{}) *MockClientInterface_GetQuery_Call {
- return &MockClientInterface_GetQuery_Call{Call: _e.mock.On("GetQuery",
- append([]interface{}{ctx, id}, reqEditors...)...)}
-}
-
-func (_c *MockClientInterface_GetQuery_Call) Run(run func(ctx context.Context, id queryapi.QueryID, reqEditors ...queryapi.RequestEditorFn)) *MockClientInterface_GetQuery_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-2)
- for i, a := range args[2:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(queryapi.QueryID), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientInterface_GetQuery_Call) Return(_a0 *http.Response, _a1 error) *MockClientInterface_GetQuery_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientInterface_GetQuery_Call) RunAndReturn(run func(context.Context, queryapi.QueryID, ...queryapi.RequestEditorFn) (*http.Response, error)) *MockClientInterface_GetQuery_Call {
- _c.Call.Return(run)
- return _c
-}
-
// GetStatusByClientId provides a mock function with given fields: ctx, id, reqEditors
func (_m *MockClientInterface) GetStatusByClientId(ctx context.Context, id queryapi.ClientID, reqEditors ...queryapi.RequestEditorFn) (*http.Response, error) {
_va := make([]interface{}, len(reqEditors))
@@ -2927,79 +2704,6 @@ func (_c *MockClientInterface_ListDocumentsByClientId_Call) RunAndReturn(run fun
return _c
}
-// ListQueries provides a mock function with given fields: ctx, reqEditors
-func (_m *MockClientInterface) ListQueries(ctx context.Context, reqEditors ...queryapi.RequestEditorFn) (*http.Response, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for ListQueries")
- }
-
- var r0 *http.Response
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, ...queryapi.RequestEditorFn) (*http.Response, error)); ok {
- return rf(ctx, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, ...queryapi.RequestEditorFn) *http.Response); ok {
- r0 = rf(ctx, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*http.Response)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientInterface_ListQueries_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListQueries'
-type MockClientInterface_ListQueries_Call struct {
- *mock.Call
-}
-
-// ListQueries is a helper method to define mock.On call
-// - ctx context.Context
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientInterface_Expecter) ListQueries(ctx interface{}, reqEditors ...interface{}) *MockClientInterface_ListQueries_Call {
- return &MockClientInterface_ListQueries_Call{Call: _e.mock.On("ListQueries",
- append([]interface{}{ctx}, reqEditors...)...)}
-}
-
-func (_c *MockClientInterface_ListQueries_Call) Run(run func(ctx context.Context, reqEditors ...queryapi.RequestEditorFn)) *MockClientInterface_ListQueries_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-1)
- for i, a := range args[1:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientInterface_ListQueries_Call) Return(_a0 *http.Response, _a1 error) *MockClientInterface_ListQueries_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientInterface_ListQueries_Call) RunAndReturn(run func(context.Context, ...queryapi.RequestEditorFn) (*http.Response, error)) *MockClientInterface_ListQueries_Call {
- _c.Call.Return(run)
- return _c
-}
-
// Login provides a mock function with given fields: ctx, reqEditors
func (_m *MockClientInterface) Login(ctx context.Context, reqEditors ...queryapi.RequestEditorFn) (*http.Response, error) {
_va := make([]interface{}, len(reqEditors))
@@ -3522,157 +3226,6 @@ func (_c *MockClientInterface_SetCollectorByClientIdWithBody_Call) RunAndReturn(
return _c
}
-// TestQuery provides a mock function with given fields: ctx, id, body, reqEditors
-func (_m *MockClientInterface) TestQuery(ctx context.Context, id queryapi.QueryID, body queryapi.TestQueryJSONRequestBody, reqEditors ...queryapi.RequestEditorFn) (*http.Response, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, id, body)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for TestQuery")
- }
-
- var r0 *http.Response
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, queryapi.TestQueryJSONRequestBody, ...queryapi.RequestEditorFn) (*http.Response, error)); ok {
- return rf(ctx, id, body, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, queryapi.TestQueryJSONRequestBody, ...queryapi.RequestEditorFn) *http.Response); ok {
- r0 = rf(ctx, id, body, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*http.Response)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, queryapi.QueryID, queryapi.TestQueryJSONRequestBody, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, id, body, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientInterface_TestQuery_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'TestQuery'
-type MockClientInterface_TestQuery_Call struct {
- *mock.Call
-}
-
-// TestQuery is a helper method to define mock.On call
-// - ctx context.Context
-// - id queryapi.QueryID
-// - body queryapi.TestQueryJSONRequestBody
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientInterface_Expecter) TestQuery(ctx interface{}, id interface{}, body interface{}, reqEditors ...interface{}) *MockClientInterface_TestQuery_Call {
- return &MockClientInterface_TestQuery_Call{Call: _e.mock.On("TestQuery",
- append([]interface{}{ctx, id, body}, reqEditors...)...)}
-}
-
-func (_c *MockClientInterface_TestQuery_Call) Run(run func(ctx context.Context, id queryapi.QueryID, body queryapi.TestQueryJSONRequestBody, reqEditors ...queryapi.RequestEditorFn)) *MockClientInterface_TestQuery_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-3)
- for i, a := range args[3:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(queryapi.QueryID), args[2].(queryapi.TestQueryJSONRequestBody), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientInterface_TestQuery_Call) Return(_a0 *http.Response, _a1 error) *MockClientInterface_TestQuery_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientInterface_TestQuery_Call) RunAndReturn(run func(context.Context, queryapi.QueryID, queryapi.TestQueryJSONRequestBody, ...queryapi.RequestEditorFn) (*http.Response, error)) *MockClientInterface_TestQuery_Call {
- _c.Call.Return(run)
- return _c
-}
-
-// TestQueryWithBody provides a mock function with given fields: ctx, id, contentType, body, reqEditors
-func (_m *MockClientInterface) TestQueryWithBody(ctx context.Context, id queryapi.QueryID, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn) (*http.Response, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, id, contentType, body)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for TestQueryWithBody")
- }
-
- var r0 *http.Response
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) (*http.Response, error)); ok {
- return rf(ctx, id, contentType, body, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) *http.Response); ok {
- r0 = rf(ctx, id, contentType, body, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*http.Response)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, id, contentType, body, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientInterface_TestQueryWithBody_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'TestQueryWithBody'
-type MockClientInterface_TestQueryWithBody_Call struct {
- *mock.Call
-}
-
-// TestQueryWithBody is a helper method to define mock.On call
-// - ctx context.Context
-// - id queryapi.QueryID
-// - contentType string
-// - body io.Reader
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientInterface_Expecter) TestQueryWithBody(ctx interface{}, id interface{}, contentType interface{}, body interface{}, reqEditors ...interface{}) *MockClientInterface_TestQueryWithBody_Call {
- return &MockClientInterface_TestQueryWithBody_Call{Call: _e.mock.On("TestQueryWithBody",
- append([]interface{}{ctx, id, contentType, body}, reqEditors...)...)}
-}
-
-func (_c *MockClientInterface_TestQueryWithBody_Call) Run(run func(ctx context.Context, id queryapi.QueryID, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn)) *MockClientInterface_TestQueryWithBody_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-4)
- for i, a := range args[4:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(queryapi.QueryID), args[2].(string), args[3].(io.Reader), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientInterface_TestQueryWithBody_Call) Return(_a0 *http.Response, _a1 error) *MockClientInterface_TestQueryWithBody_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientInterface_TestQueryWithBody_Call) RunAndReturn(run func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) (*http.Response, error)) *MockClientInterface_TestQueryWithBody_Call {
- _c.Call.Return(run)
- return _c
-}
-
// TriggerExport provides a mock function with given fields: ctx, id, body, reqEditors
func (_m *MockClientInterface) TriggerExport(ctx context.Context, id queryapi.ClientID, body queryapi.TriggerExportJSONRequestBody, reqEditors ...queryapi.RequestEditorFn) (*http.Response, error) {
_va := make([]interface{}, len(reqEditors))
@@ -4126,157 +3679,6 @@ func (_c *MockClientInterface_UpdateClientWithBody_Call) RunAndReturn(run func(c
return _c
}
-// UpdateQuery provides a mock function with given fields: ctx, id, body, reqEditors
-func (_m *MockClientInterface) UpdateQuery(ctx context.Context, id queryapi.QueryID, body queryapi.UpdateQueryJSONRequestBody, reqEditors ...queryapi.RequestEditorFn) (*http.Response, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, id, body)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for UpdateQuery")
- }
-
- var r0 *http.Response
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, queryapi.UpdateQueryJSONRequestBody, ...queryapi.RequestEditorFn) (*http.Response, error)); ok {
- return rf(ctx, id, body, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, queryapi.UpdateQueryJSONRequestBody, ...queryapi.RequestEditorFn) *http.Response); ok {
- r0 = rf(ctx, id, body, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*http.Response)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, queryapi.QueryID, queryapi.UpdateQueryJSONRequestBody, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, id, body, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientInterface_UpdateQuery_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateQuery'
-type MockClientInterface_UpdateQuery_Call struct {
- *mock.Call
-}
-
-// UpdateQuery is a helper method to define mock.On call
-// - ctx context.Context
-// - id queryapi.QueryID
-// - body queryapi.UpdateQueryJSONRequestBody
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientInterface_Expecter) UpdateQuery(ctx interface{}, id interface{}, body interface{}, reqEditors ...interface{}) *MockClientInterface_UpdateQuery_Call {
- return &MockClientInterface_UpdateQuery_Call{Call: _e.mock.On("UpdateQuery",
- append([]interface{}{ctx, id, body}, reqEditors...)...)}
-}
-
-func (_c *MockClientInterface_UpdateQuery_Call) Run(run func(ctx context.Context, id queryapi.QueryID, body queryapi.UpdateQueryJSONRequestBody, reqEditors ...queryapi.RequestEditorFn)) *MockClientInterface_UpdateQuery_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-3)
- for i, a := range args[3:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(queryapi.QueryID), args[2].(queryapi.UpdateQueryJSONRequestBody), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientInterface_UpdateQuery_Call) Return(_a0 *http.Response, _a1 error) *MockClientInterface_UpdateQuery_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientInterface_UpdateQuery_Call) RunAndReturn(run func(context.Context, queryapi.QueryID, queryapi.UpdateQueryJSONRequestBody, ...queryapi.RequestEditorFn) (*http.Response, error)) *MockClientInterface_UpdateQuery_Call {
- _c.Call.Return(run)
- return _c
-}
-
-// UpdateQueryWithBody provides a mock function with given fields: ctx, id, contentType, body, reqEditors
-func (_m *MockClientInterface) UpdateQueryWithBody(ctx context.Context, id queryapi.QueryID, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn) (*http.Response, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, id, contentType, body)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for UpdateQueryWithBody")
- }
-
- var r0 *http.Response
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) (*http.Response, error)); ok {
- return rf(ctx, id, contentType, body, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) *http.Response); ok {
- r0 = rf(ctx, id, contentType, body, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*http.Response)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, id, contentType, body, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientInterface_UpdateQueryWithBody_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateQueryWithBody'
-type MockClientInterface_UpdateQueryWithBody_Call struct {
- *mock.Call
-}
-
-// UpdateQueryWithBody is a helper method to define mock.On call
-// - ctx context.Context
-// - id queryapi.QueryID
-// - contentType string
-// - body io.Reader
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientInterface_Expecter) UpdateQueryWithBody(ctx interface{}, id interface{}, contentType interface{}, body interface{}, reqEditors ...interface{}) *MockClientInterface_UpdateQueryWithBody_Call {
- return &MockClientInterface_UpdateQueryWithBody_Call{Call: _e.mock.On("UpdateQueryWithBody",
- append([]interface{}{ctx, id, contentType, body}, reqEditors...)...)}
-}
-
-func (_c *MockClientInterface_UpdateQueryWithBody_Call) Run(run func(ctx context.Context, id queryapi.QueryID, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn)) *MockClientInterface_UpdateQueryWithBody_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-4)
- for i, a := range args[4:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(queryapi.QueryID), args[2].(string), args[3].(io.Reader), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientInterface_UpdateQueryWithBody_Call) Return(_a0 *http.Response, _a1 error) *MockClientInterface_UpdateQueryWithBody_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientInterface_UpdateQueryWithBody_Call) RunAndReturn(run func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) (*http.Response, error)) *MockClientInterface_UpdateQueryWithBody_Call {
- _c.Call.Return(run)
- return _c
-}
-
// UploadDocumentBatchWithBody provides a mock function with given fields: ctx, id, contentType, body, reqEditors
func (_m *MockClientInterface) UploadDocumentBatchWithBody(ctx context.Context, id queryapi.ClientID, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn) (*http.Response, error) {
_va := make([]interface{}, len(reqEditors))
diff --git a/mocks/queryapi/mock_ClientWithResponsesInterface.go b/mocks/queryapi/mock_ClientWithResponsesInterface.go
index af80dc30..0649ffc8 100644
--- a/mocks/queryapi/mock_ClientWithResponsesInterface.go
+++ b/mocks/queryapi/mock_ClientWithResponsesInterface.go
@@ -922,155 +922,6 @@ func (_c *MockClientWithResponsesInterface_CreateFolderWithResponse_Call) RunAnd
return _c
}
-// CreateQueryWithBodyWithResponse provides a mock function with given fields: ctx, contentType, body, reqEditors
-func (_m *MockClientWithResponsesInterface) CreateQueryWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn) (*queryapi.CreateQueryResponse, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, contentType, body)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for CreateQueryWithBodyWithResponse")
- }
-
- var r0 *queryapi.CreateQueryResponse
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, string, io.Reader, ...queryapi.RequestEditorFn) (*queryapi.CreateQueryResponse, error)); ok {
- return rf(ctx, contentType, body, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, string, io.Reader, ...queryapi.RequestEditorFn) *queryapi.CreateQueryResponse); ok {
- r0 = rf(ctx, contentType, body, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*queryapi.CreateQueryResponse)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, string, io.Reader, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, contentType, body, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientWithResponsesInterface_CreateQueryWithBodyWithResponse_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateQueryWithBodyWithResponse'
-type MockClientWithResponsesInterface_CreateQueryWithBodyWithResponse_Call struct {
- *mock.Call
-}
-
-// CreateQueryWithBodyWithResponse is a helper method to define mock.On call
-// - ctx context.Context
-// - contentType string
-// - body io.Reader
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientWithResponsesInterface_Expecter) CreateQueryWithBodyWithResponse(ctx interface{}, contentType interface{}, body interface{}, reqEditors ...interface{}) *MockClientWithResponsesInterface_CreateQueryWithBodyWithResponse_Call {
- return &MockClientWithResponsesInterface_CreateQueryWithBodyWithResponse_Call{Call: _e.mock.On("CreateQueryWithBodyWithResponse",
- append([]interface{}{ctx, contentType, body}, reqEditors...)...)}
-}
-
-func (_c *MockClientWithResponsesInterface_CreateQueryWithBodyWithResponse_Call) Run(run func(ctx context.Context, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn)) *MockClientWithResponsesInterface_CreateQueryWithBodyWithResponse_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-3)
- for i, a := range args[3:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(string), args[2].(io.Reader), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_CreateQueryWithBodyWithResponse_Call) Return(_a0 *queryapi.CreateQueryResponse, _a1 error) *MockClientWithResponsesInterface_CreateQueryWithBodyWithResponse_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_CreateQueryWithBodyWithResponse_Call) RunAndReturn(run func(context.Context, string, io.Reader, ...queryapi.RequestEditorFn) (*queryapi.CreateQueryResponse, error)) *MockClientWithResponsesInterface_CreateQueryWithBodyWithResponse_Call {
- _c.Call.Return(run)
- return _c
-}
-
-// CreateQueryWithResponse provides a mock function with given fields: ctx, body, reqEditors
-func (_m *MockClientWithResponsesInterface) CreateQueryWithResponse(ctx context.Context, body queryapi.CreateQueryJSONRequestBody, reqEditors ...queryapi.RequestEditorFn) (*queryapi.CreateQueryResponse, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, body)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for CreateQueryWithResponse")
- }
-
- var r0 *queryapi.CreateQueryResponse
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.CreateQueryJSONRequestBody, ...queryapi.RequestEditorFn) (*queryapi.CreateQueryResponse, error)); ok {
- return rf(ctx, body, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.CreateQueryJSONRequestBody, ...queryapi.RequestEditorFn) *queryapi.CreateQueryResponse); ok {
- r0 = rf(ctx, body, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*queryapi.CreateQueryResponse)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, queryapi.CreateQueryJSONRequestBody, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, body, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientWithResponsesInterface_CreateQueryWithResponse_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateQueryWithResponse'
-type MockClientWithResponsesInterface_CreateQueryWithResponse_Call struct {
- *mock.Call
-}
-
-// CreateQueryWithResponse is a helper method to define mock.On call
-// - ctx context.Context
-// - body queryapi.CreateQueryJSONRequestBody
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientWithResponsesInterface_Expecter) CreateQueryWithResponse(ctx interface{}, body interface{}, reqEditors ...interface{}) *MockClientWithResponsesInterface_CreateQueryWithResponse_Call {
- return &MockClientWithResponsesInterface_CreateQueryWithResponse_Call{Call: _e.mock.On("CreateQueryWithResponse",
- append([]interface{}{ctx, body}, reqEditors...)...)}
-}
-
-func (_c *MockClientWithResponsesInterface_CreateQueryWithResponse_Call) Run(run func(ctx context.Context, body queryapi.CreateQueryJSONRequestBody, reqEditors ...queryapi.RequestEditorFn)) *MockClientWithResponsesInterface_CreateQueryWithResponse_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-2)
- for i, a := range args[2:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(queryapi.CreateQueryJSONRequestBody), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_CreateQueryWithResponse_Call) Return(_a0 *queryapi.CreateQueryResponse, _a1 error) *MockClientWithResponsesInterface_CreateQueryWithResponse_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_CreateQueryWithResponse_Call) RunAndReturn(run func(context.Context, queryapi.CreateQueryJSONRequestBody, ...queryapi.RequestEditorFn) (*queryapi.CreateQueryResponse, error)) *MockClientWithResponsesInterface_CreateQueryWithResponse_Call {
- _c.Call.Return(run)
- return _c
-}
-
// DeleteAdminUserWithResponse provides a mock function with given fields: ctx, email, params, reqEditors
func (_m *MockClientWithResponsesInterface) DeleteAdminUserWithResponse(ctx context.Context, email queryapi.UserEmail, params *queryapi.DeleteAdminUserParams, reqEditors ...queryapi.RequestEditorFn) (*queryapi.DeleteAdminUserResponse, error) {
_va := make([]interface{}, len(reqEditors))
@@ -2406,80 +2257,6 @@ func (_c *MockClientWithResponsesInterface_GetIdentityWithResponse_Call) RunAndR
return _c
}
-// GetQueryWithResponse provides a mock function with given fields: ctx, id, reqEditors
-func (_m *MockClientWithResponsesInterface) GetQueryWithResponse(ctx context.Context, id queryapi.QueryID, reqEditors ...queryapi.RequestEditorFn) (*queryapi.GetQueryResponse, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, id)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for GetQueryWithResponse")
- }
-
- var r0 *queryapi.GetQueryResponse
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, ...queryapi.RequestEditorFn) (*queryapi.GetQueryResponse, error)); ok {
- return rf(ctx, id, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, ...queryapi.RequestEditorFn) *queryapi.GetQueryResponse); ok {
- r0 = rf(ctx, id, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*queryapi.GetQueryResponse)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, queryapi.QueryID, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, id, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientWithResponsesInterface_GetQueryWithResponse_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetQueryWithResponse'
-type MockClientWithResponsesInterface_GetQueryWithResponse_Call struct {
- *mock.Call
-}
-
-// GetQueryWithResponse is a helper method to define mock.On call
-// - ctx context.Context
-// - id queryapi.QueryID
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientWithResponsesInterface_Expecter) GetQueryWithResponse(ctx interface{}, id interface{}, reqEditors ...interface{}) *MockClientWithResponsesInterface_GetQueryWithResponse_Call {
- return &MockClientWithResponsesInterface_GetQueryWithResponse_Call{Call: _e.mock.On("GetQueryWithResponse",
- append([]interface{}{ctx, id}, reqEditors...)...)}
-}
-
-func (_c *MockClientWithResponsesInterface_GetQueryWithResponse_Call) Run(run func(ctx context.Context, id queryapi.QueryID, reqEditors ...queryapi.RequestEditorFn)) *MockClientWithResponsesInterface_GetQueryWithResponse_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-2)
- for i, a := range args[2:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(queryapi.QueryID), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_GetQueryWithResponse_Call) Return(_a0 *queryapi.GetQueryResponse, _a1 error) *MockClientWithResponsesInterface_GetQueryWithResponse_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_GetQueryWithResponse_Call) RunAndReturn(run func(context.Context, queryapi.QueryID, ...queryapi.RequestEditorFn) (*queryapi.GetQueryResponse, error)) *MockClientWithResponsesInterface_GetQueryWithResponse_Call {
- _c.Call.Return(run)
- return _c
-}
-
// GetStatusByClientIdWithResponse provides a mock function with given fields: ctx, id, reqEditors
func (_m *MockClientWithResponsesInterface) GetStatusByClientIdWithResponse(ctx context.Context, id queryapi.ClientID, reqEditors ...queryapi.RequestEditorFn) (*queryapi.GetStatusByClientIdResponse, error) {
_va := make([]interface{}, len(reqEditors))
@@ -2925,79 +2702,6 @@ func (_c *MockClientWithResponsesInterface_ListDocumentsByClientIdWithResponse_C
return _c
}
-// ListQueriesWithResponse provides a mock function with given fields: ctx, reqEditors
-func (_m *MockClientWithResponsesInterface) ListQueriesWithResponse(ctx context.Context, reqEditors ...queryapi.RequestEditorFn) (*queryapi.ListQueriesResponse, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for ListQueriesWithResponse")
- }
-
- var r0 *queryapi.ListQueriesResponse
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, ...queryapi.RequestEditorFn) (*queryapi.ListQueriesResponse, error)); ok {
- return rf(ctx, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, ...queryapi.RequestEditorFn) *queryapi.ListQueriesResponse); ok {
- r0 = rf(ctx, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*queryapi.ListQueriesResponse)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientWithResponsesInterface_ListQueriesWithResponse_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListQueriesWithResponse'
-type MockClientWithResponsesInterface_ListQueriesWithResponse_Call struct {
- *mock.Call
-}
-
-// ListQueriesWithResponse is a helper method to define mock.On call
-// - ctx context.Context
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientWithResponsesInterface_Expecter) ListQueriesWithResponse(ctx interface{}, reqEditors ...interface{}) *MockClientWithResponsesInterface_ListQueriesWithResponse_Call {
- return &MockClientWithResponsesInterface_ListQueriesWithResponse_Call{Call: _e.mock.On("ListQueriesWithResponse",
- append([]interface{}{ctx}, reqEditors...)...)}
-}
-
-func (_c *MockClientWithResponsesInterface_ListQueriesWithResponse_Call) Run(run func(ctx context.Context, reqEditors ...queryapi.RequestEditorFn)) *MockClientWithResponsesInterface_ListQueriesWithResponse_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-1)
- for i, a := range args[1:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_ListQueriesWithResponse_Call) Return(_a0 *queryapi.ListQueriesResponse, _a1 error) *MockClientWithResponsesInterface_ListQueriesWithResponse_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_ListQueriesWithResponse_Call) RunAndReturn(run func(context.Context, ...queryapi.RequestEditorFn) (*queryapi.ListQueriesResponse, error)) *MockClientWithResponsesInterface_ListQueriesWithResponse_Call {
- _c.Call.Return(run)
- return _c
-}
-
// LoginCallbackWithResponse provides a mock function with given fields: ctx, params, reqEditors
func (_m *MockClientWithResponsesInterface) LoginCallbackWithResponse(ctx context.Context, params *queryapi.LoginCallbackParams, reqEditors ...queryapi.RequestEditorFn) (*queryapi.LoginCallbackResponse, error) {
_va := make([]interface{}, len(reqEditors))
@@ -3520,157 +3224,6 @@ func (_c *MockClientWithResponsesInterface_SetCollectorByClientIdWithResponse_Ca
return _c
}
-// TestQueryWithBodyWithResponse provides a mock function with given fields: ctx, id, contentType, body, reqEditors
-func (_m *MockClientWithResponsesInterface) TestQueryWithBodyWithResponse(ctx context.Context, id queryapi.QueryID, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn) (*queryapi.TestQueryResponse, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, id, contentType, body)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for TestQueryWithBodyWithResponse")
- }
-
- var r0 *queryapi.TestQueryResponse
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) (*queryapi.TestQueryResponse, error)); ok {
- return rf(ctx, id, contentType, body, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) *queryapi.TestQueryResponse); ok {
- r0 = rf(ctx, id, contentType, body, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*queryapi.TestQueryResponse)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, id, contentType, body, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientWithResponsesInterface_TestQueryWithBodyWithResponse_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'TestQueryWithBodyWithResponse'
-type MockClientWithResponsesInterface_TestQueryWithBodyWithResponse_Call struct {
- *mock.Call
-}
-
-// TestQueryWithBodyWithResponse is a helper method to define mock.On call
-// - ctx context.Context
-// - id queryapi.QueryID
-// - contentType string
-// - body io.Reader
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientWithResponsesInterface_Expecter) TestQueryWithBodyWithResponse(ctx interface{}, id interface{}, contentType interface{}, body interface{}, reqEditors ...interface{}) *MockClientWithResponsesInterface_TestQueryWithBodyWithResponse_Call {
- return &MockClientWithResponsesInterface_TestQueryWithBodyWithResponse_Call{Call: _e.mock.On("TestQueryWithBodyWithResponse",
- append([]interface{}{ctx, id, contentType, body}, reqEditors...)...)}
-}
-
-func (_c *MockClientWithResponsesInterface_TestQueryWithBodyWithResponse_Call) Run(run func(ctx context.Context, id queryapi.QueryID, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn)) *MockClientWithResponsesInterface_TestQueryWithBodyWithResponse_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-4)
- for i, a := range args[4:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(queryapi.QueryID), args[2].(string), args[3].(io.Reader), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_TestQueryWithBodyWithResponse_Call) Return(_a0 *queryapi.TestQueryResponse, _a1 error) *MockClientWithResponsesInterface_TestQueryWithBodyWithResponse_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_TestQueryWithBodyWithResponse_Call) RunAndReturn(run func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) (*queryapi.TestQueryResponse, error)) *MockClientWithResponsesInterface_TestQueryWithBodyWithResponse_Call {
- _c.Call.Return(run)
- return _c
-}
-
-// TestQueryWithResponse provides a mock function with given fields: ctx, id, body, reqEditors
-func (_m *MockClientWithResponsesInterface) TestQueryWithResponse(ctx context.Context, id queryapi.QueryID, body queryapi.TestQueryJSONRequestBody, reqEditors ...queryapi.RequestEditorFn) (*queryapi.TestQueryResponse, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, id, body)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for TestQueryWithResponse")
- }
-
- var r0 *queryapi.TestQueryResponse
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, queryapi.TestQueryJSONRequestBody, ...queryapi.RequestEditorFn) (*queryapi.TestQueryResponse, error)); ok {
- return rf(ctx, id, body, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, queryapi.TestQueryJSONRequestBody, ...queryapi.RequestEditorFn) *queryapi.TestQueryResponse); ok {
- r0 = rf(ctx, id, body, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*queryapi.TestQueryResponse)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, queryapi.QueryID, queryapi.TestQueryJSONRequestBody, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, id, body, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientWithResponsesInterface_TestQueryWithResponse_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'TestQueryWithResponse'
-type MockClientWithResponsesInterface_TestQueryWithResponse_Call struct {
- *mock.Call
-}
-
-// TestQueryWithResponse is a helper method to define mock.On call
-// - ctx context.Context
-// - id queryapi.QueryID
-// - body queryapi.TestQueryJSONRequestBody
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientWithResponsesInterface_Expecter) TestQueryWithResponse(ctx interface{}, id interface{}, body interface{}, reqEditors ...interface{}) *MockClientWithResponsesInterface_TestQueryWithResponse_Call {
- return &MockClientWithResponsesInterface_TestQueryWithResponse_Call{Call: _e.mock.On("TestQueryWithResponse",
- append([]interface{}{ctx, id, body}, reqEditors...)...)}
-}
-
-func (_c *MockClientWithResponsesInterface_TestQueryWithResponse_Call) Run(run func(ctx context.Context, id queryapi.QueryID, body queryapi.TestQueryJSONRequestBody, reqEditors ...queryapi.RequestEditorFn)) *MockClientWithResponsesInterface_TestQueryWithResponse_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-3)
- for i, a := range args[3:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(queryapi.QueryID), args[2].(queryapi.TestQueryJSONRequestBody), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_TestQueryWithResponse_Call) Return(_a0 *queryapi.TestQueryResponse, _a1 error) *MockClientWithResponsesInterface_TestQueryWithResponse_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_TestQueryWithResponse_Call) RunAndReturn(run func(context.Context, queryapi.QueryID, queryapi.TestQueryJSONRequestBody, ...queryapi.RequestEditorFn) (*queryapi.TestQueryResponse, error)) *MockClientWithResponsesInterface_TestQueryWithResponse_Call {
- _c.Call.Return(run)
- return _c
-}
-
// TriggerExportWithBodyWithResponse provides a mock function with given fields: ctx, id, contentType, body, reqEditors
func (_m *MockClientWithResponsesInterface) TriggerExportWithBodyWithResponse(ctx context.Context, id queryapi.ClientID, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn) (*queryapi.TriggerExportResponse, error) {
_va := make([]interface{}, len(reqEditors))
@@ -4124,157 +3677,6 @@ func (_c *MockClientWithResponsesInterface_UpdateClientWithResponse_Call) RunAnd
return _c
}
-// UpdateQueryWithBodyWithResponse provides a mock function with given fields: ctx, id, contentType, body, reqEditors
-func (_m *MockClientWithResponsesInterface) UpdateQueryWithBodyWithResponse(ctx context.Context, id queryapi.QueryID, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn) (*queryapi.UpdateQueryResponse, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, id, contentType, body)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for UpdateQueryWithBodyWithResponse")
- }
-
- var r0 *queryapi.UpdateQueryResponse
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) (*queryapi.UpdateQueryResponse, error)); ok {
- return rf(ctx, id, contentType, body, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) *queryapi.UpdateQueryResponse); ok {
- r0 = rf(ctx, id, contentType, body, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*queryapi.UpdateQueryResponse)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, id, contentType, body, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientWithResponsesInterface_UpdateQueryWithBodyWithResponse_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateQueryWithBodyWithResponse'
-type MockClientWithResponsesInterface_UpdateQueryWithBodyWithResponse_Call struct {
- *mock.Call
-}
-
-// UpdateQueryWithBodyWithResponse is a helper method to define mock.On call
-// - ctx context.Context
-// - id queryapi.QueryID
-// - contentType string
-// - body io.Reader
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientWithResponsesInterface_Expecter) UpdateQueryWithBodyWithResponse(ctx interface{}, id interface{}, contentType interface{}, body interface{}, reqEditors ...interface{}) *MockClientWithResponsesInterface_UpdateQueryWithBodyWithResponse_Call {
- return &MockClientWithResponsesInterface_UpdateQueryWithBodyWithResponse_Call{Call: _e.mock.On("UpdateQueryWithBodyWithResponse",
- append([]interface{}{ctx, id, contentType, body}, reqEditors...)...)}
-}
-
-func (_c *MockClientWithResponsesInterface_UpdateQueryWithBodyWithResponse_Call) Run(run func(ctx context.Context, id queryapi.QueryID, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn)) *MockClientWithResponsesInterface_UpdateQueryWithBodyWithResponse_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-4)
- for i, a := range args[4:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(queryapi.QueryID), args[2].(string), args[3].(io.Reader), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_UpdateQueryWithBodyWithResponse_Call) Return(_a0 *queryapi.UpdateQueryResponse, _a1 error) *MockClientWithResponsesInterface_UpdateQueryWithBodyWithResponse_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_UpdateQueryWithBodyWithResponse_Call) RunAndReturn(run func(context.Context, queryapi.QueryID, string, io.Reader, ...queryapi.RequestEditorFn) (*queryapi.UpdateQueryResponse, error)) *MockClientWithResponsesInterface_UpdateQueryWithBodyWithResponse_Call {
- _c.Call.Return(run)
- return _c
-}
-
-// UpdateQueryWithResponse provides a mock function with given fields: ctx, id, body, reqEditors
-func (_m *MockClientWithResponsesInterface) UpdateQueryWithResponse(ctx context.Context, id queryapi.QueryID, body queryapi.UpdateQueryJSONRequestBody, reqEditors ...queryapi.RequestEditorFn) (*queryapi.UpdateQueryResponse, error) {
- _va := make([]interface{}, len(reqEditors))
- for _i := range reqEditors {
- _va[_i] = reqEditors[_i]
- }
- var _ca []interface{}
- _ca = append(_ca, ctx, id, body)
- _ca = append(_ca, _va...)
- ret := _m.Called(_ca...)
-
- if len(ret) == 0 {
- panic("no return value specified for UpdateQueryWithResponse")
- }
-
- var r0 *queryapi.UpdateQueryResponse
- var r1 error
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, queryapi.UpdateQueryJSONRequestBody, ...queryapi.RequestEditorFn) (*queryapi.UpdateQueryResponse, error)); ok {
- return rf(ctx, id, body, reqEditors...)
- }
- if rf, ok := ret.Get(0).(func(context.Context, queryapi.QueryID, queryapi.UpdateQueryJSONRequestBody, ...queryapi.RequestEditorFn) *queryapi.UpdateQueryResponse); ok {
- r0 = rf(ctx, id, body, reqEditors...)
- } else {
- if ret.Get(0) != nil {
- r0 = ret.Get(0).(*queryapi.UpdateQueryResponse)
- }
- }
-
- if rf, ok := ret.Get(1).(func(context.Context, queryapi.QueryID, queryapi.UpdateQueryJSONRequestBody, ...queryapi.RequestEditorFn) error); ok {
- r1 = rf(ctx, id, body, reqEditors...)
- } else {
- r1 = ret.Error(1)
- }
-
- return r0, r1
-}
-
-// MockClientWithResponsesInterface_UpdateQueryWithResponse_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateQueryWithResponse'
-type MockClientWithResponsesInterface_UpdateQueryWithResponse_Call struct {
- *mock.Call
-}
-
-// UpdateQueryWithResponse is a helper method to define mock.On call
-// - ctx context.Context
-// - id queryapi.QueryID
-// - body queryapi.UpdateQueryJSONRequestBody
-// - reqEditors ...queryapi.RequestEditorFn
-func (_e *MockClientWithResponsesInterface_Expecter) UpdateQueryWithResponse(ctx interface{}, id interface{}, body interface{}, reqEditors ...interface{}) *MockClientWithResponsesInterface_UpdateQueryWithResponse_Call {
- return &MockClientWithResponsesInterface_UpdateQueryWithResponse_Call{Call: _e.mock.On("UpdateQueryWithResponse",
- append([]interface{}{ctx, id, body}, reqEditors...)...)}
-}
-
-func (_c *MockClientWithResponsesInterface_UpdateQueryWithResponse_Call) Run(run func(ctx context.Context, id queryapi.QueryID, body queryapi.UpdateQueryJSONRequestBody, reqEditors ...queryapi.RequestEditorFn)) *MockClientWithResponsesInterface_UpdateQueryWithResponse_Call {
- _c.Call.Run(func(args mock.Arguments) {
- variadicArgs := make([]queryapi.RequestEditorFn, len(args)-3)
- for i, a := range args[3:] {
- if a != nil {
- variadicArgs[i] = a.(queryapi.RequestEditorFn)
- }
- }
- run(args[0].(context.Context), args[1].(queryapi.QueryID), args[2].(queryapi.UpdateQueryJSONRequestBody), variadicArgs...)
- })
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_UpdateQueryWithResponse_Call) Return(_a0 *queryapi.UpdateQueryResponse, _a1 error) *MockClientWithResponsesInterface_UpdateQueryWithResponse_Call {
- _c.Call.Return(_a0, _a1)
- return _c
-}
-
-func (_c *MockClientWithResponsesInterface_UpdateQueryWithResponse_Call) RunAndReturn(run func(context.Context, queryapi.QueryID, queryapi.UpdateQueryJSONRequestBody, ...queryapi.RequestEditorFn) (*queryapi.UpdateQueryResponse, error)) *MockClientWithResponsesInterface_UpdateQueryWithResponse_Call {
- _c.Call.Return(run)
- return _c
-}
-
// UploadDocumentBatchWithBodyWithResponse provides a mock function with given fields: ctx, id, contentType, body, reqEditors
func (_m *MockClientWithResponsesInterface) UploadDocumentBatchWithBodyWithResponse(ctx context.Context, id queryapi.ClientID, contentType string, body io.Reader, reqEditors ...queryapi.RequestEditorFn) (*queryapi.UploadDocumentBatchResponse, error) {
_va := make([]interface{}, len(reqEditors))
diff --git a/pkg/queryAPI/api.gen.go b/pkg/queryAPI/api.gen.go
index 49e8610a..42ab6d2b 100644
--- a/pkg/queryAPI/api.gen.go
+++ b/pkg/queryAPI/api.gen.go
@@ -64,12 +64,6 @@ const (
RightClosedInterval FieldFilterCondition = "right_closed_interval"
)
-// Defines values for QueryType.
-const (
- CONTEXTFULL QueryType = "CONTEXT_FULL"
- JSONEXTRACTOR QueryType = "JSON_EXTRACTOR"
-)
-
// Defines values for ListAdminUsersParamsStatus.
const (
All ListAdminUsersParamsStatus = "all"
@@ -519,9 +513,6 @@ type Collector struct {
// ClientId The client external id
ClientId ClientID `json:"client_id"`
- // Fields The fields in the collector.
- Fields CollectorFields `json:"fields"`
-
// LatestVersion The desired version.
LatestVersion Version `json:"latest_version"`
@@ -532,29 +523,14 @@ type Collector struct {
MinimumTextVersion CodeVersion `json:"minimum_text_version"`
}
-// CollectorField The field properties for the collector.
-type CollectorField struct {
- // Name The output field name.
- Name CollectorFieldName `json:"name"`
-
- // QueryId The query id.
- QueryId QueryID `json:"query_id"`
-}
-
// CollectorFieldName The output field name.
type CollectorFieldName = string
-// CollectorFields The fields in the collector.
-type CollectorFields = []CollectorField
-
// CollectorSet Payload for updating a Collector.
type CollectorSet struct {
// ActiveVersion The desired version.
ActiveVersion *Version `json:"active_version,omitempty"`
- // Fields The fields in the collector.
- Fields *CollectorFields `json:"fields,omitempty"`
-
// MinimumCleanerVersion The desired code version.
MinimumCleanerVersion *CodeVersion `json:"minimum_cleaner_version,omitempty"`
@@ -579,9 +555,6 @@ type DocumentEnriched struct {
// ClientId The client external id
ClientId ClientID `json:"client_id"`
- // Fields The fields and the value for the document
- Fields map[string]interface{} `json:"fields"`
-
// FileSizeBytes File size in bytes (null for legacy documents)
FileSizeBytes *int64 `json:"fileSizeBytes,omitempty"`
@@ -898,84 +871,6 @@ type LabelRecord struct {
// ListDocuments The documents in the client.
type ListDocuments = []DocumentSummary
-// ListQueries A set of queries.
-type ListQueries struct {
- // Queries List of queries.
- Queries []Query `json:"queries"`
-}
-
-// Query A logic unit of execution.
-type Query struct {
- // ActiveVersion The desired version.
- ActiveVersion Version `json:"active_version"`
-
- // Config Configuration for the query.
- Config *QueryConfig `json:"config,omitempty"`
-
- // Id The query id.
- Id QueryID `json:"id"`
-
- // LatestVersion The desired version.
- LatestVersion Version `json:"latest_version"`
-
- // RequiredQueries List of required query IDs.
- RequiredQueries *RequiredQueryIDs `json:"required_queries,omitempty"`
-
- // Type Specifies the type of the query.
- Type QueryType `json:"type"`
-}
-
-// QueryConfig Configuration for the query.
-type QueryConfig = string
-
-// QueryCreate The parameters required to create a query.
-type QueryCreate struct {
- // Config Configuration for the query.
- Config *QueryConfig `json:"config,omitempty"`
-
- // RequiredQueries List of required query IDs.
- RequiredQueries *RequiredQueryIDs `json:"required_queries,omitempty"`
-
- // Type Specifies the type of the query.
- Type QueryType `json:"type"`
-}
-
-// QueryID The query id.
-type QueryID = openapi_types.UUID
-
-// QueryTestRequest The properties for a query test request.
-type QueryTestRequest struct {
- // DocumentId The document id.
- DocumentId DocumentID `json:"document_id"`
-
- // QueryVersion The desired version.
- QueryVersion Version `json:"query_version"`
-}
-
-// QueryTestResponse The response from a query test.
-type QueryTestResponse struct {
- // Value Result of the query test.
- Value string `json:"value"`
-}
-
-// QueryType Specifies the type of the query.
-type QueryType string
-
-// QueryUpdate The properties that may be updated for a query.
-type QueryUpdate struct {
- // ActiveVersion The desired version.
- ActiveVersion *Version `json:"active_version,omitempty"`
-
- // Config Configuration for the query.
- Config *QueryConfig `json:"config,omitempty"`
-
- // RequiredQueries List of required query IDs.
- RequiredQueries *RequiredQueryIDs `json:"required_queries,omitempty"`
-}
-
-// RequiredQueryIDs List of required query IDs.
-type RequiredQueryIDs = []QueryID
-
// SingleFields Single-value fields for a document extraction (1:1 relationship)
type SingleFields struct {
// AareteDerivedAmendmentNum Amendment number derived by Aarete
@@ -1214,15 +1109,6 @@ type CreateFolderJSONRequestBody = FolderCreate
// RenameFolderJSONRequestBody defines body for RenameFolder for application/json ContentType.
type RenameFolderJSONRequestBody = FolderRename
-// CreateQueryJSONRequestBody defines body for CreateQuery for application/json ContentType.
-type CreateQueryJSONRequestBody = QueryCreate
-
-// UpdateQueryJSONRequestBody defines body for UpdateQuery for application/json ContentType.
-type UpdateQueryJSONRequestBody = QueryUpdate
-
-// TestQueryJSONRequestBody defines body for TestQuery for application/json ContentType.
-type TestQueryJSONRequestBody = QueryTestRequest
-
// RequestEditorFn is the function signature for the RequestEditor callback function
type RequestEditorFn func(ctx context.Context, req *http.Request) error
@@ -1438,27 +1324,6 @@ type ClientInterface interface {
// Logout request
Logout(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error)
-
- // ListQueries request
- ListQueries(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error)
-
- // CreateQueryWithBody request with any body
- CreateQueryWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error)
-
- CreateQuery(ctx context.Context, body CreateQueryJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error)
-
- // GetQuery request
- GetQuery(ctx context.Context, id QueryID, reqEditors ...RequestEditorFn) (*http.Response, error)
-
- // UpdateQueryWithBody request with any body
- UpdateQueryWithBody(ctx context.Context, id QueryID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error)
-
- UpdateQuery(ctx context.Context, id QueryID, body UpdateQueryJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error)
-
- // TestQueryWithBody request with any body
- TestQueryWithBody(ctx context.Context, id QueryID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error)
-
- TestQuery(ctx context.Context, id QueryID, body TestQueryJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error)
}
func (c *Client) ListAdminUsers(ctx context.Context, params *ListAdminUsersParams, reqEditors ...RequestEditorFn) (*http.Response, error) {
@@ -2073,102 +1938,6 @@ func (c *Client) Logout(ctx context.Context, reqEditors ...RequestEditorFn) (*ht
return c.Client.Do(req)
}
-func (c *Client) ListQueries(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) {
- req, err := NewListQueriesRequest(c.Server)
- if err != nil {
- return nil, err
- }
- req = req.WithContext(ctx)
- if err := c.applyEditors(ctx, req, reqEditors); err != nil {
- return nil, err
- }
- return c.Client.Do(req)
-}
-
-func (c *Client) CreateQueryWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) {
- req, err := NewCreateQueryRequestWithBody(c.Server, contentType, body)
- if err != nil {
- return nil, err
- }
- req = req.WithContext(ctx)
- if err := c.applyEditors(ctx, req, reqEditors); err != nil {
- return nil, err
- }
- return c.Client.Do(req)
-}
-
-func (c *Client) CreateQuery(ctx context.Context, body CreateQueryJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) {
- req, err := NewCreateQueryRequest(c.Server, body)
- if err != nil {
- return nil, err
- }
- req = req.WithContext(ctx)
- if err := c.applyEditors(ctx, req, reqEditors); err != nil {
- return nil, err
- }
- return c.Client.Do(req)
-}
-
-func (c *Client) GetQuery(ctx context.Context, id QueryID, reqEditors ...RequestEditorFn) (*http.Response, error) {
- req, err := NewGetQueryRequest(c.Server, id)
- if err != nil {
- return nil, err
- }
- req = req.WithContext(ctx)
- if err := c.applyEditors(ctx, req, reqEditors); err != nil {
- return nil, err
- }
- return c.Client.Do(req)
-}
-
-func (c *Client) UpdateQueryWithBody(ctx context.Context, id QueryID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) {
- req, err := NewUpdateQueryRequestWithBody(c.Server, id, contentType, body)
- if err != nil {
- return nil, err
- }
- req = req.WithContext(ctx)
- if err := c.applyEditors(ctx, req, reqEditors); err != nil {
- return nil, err
- }
- return c.Client.Do(req)
-}
-
-func (c *Client) UpdateQuery(ctx context.Context, id QueryID, body UpdateQueryJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) {
- req, err := NewUpdateQueryRequest(c.Server, id, body)
- if err != nil {
- return nil, err
- }
- req = req.WithContext(ctx)
- if err := c.applyEditors(ctx, req, reqEditors); err != nil {
- return nil, err
- }
- return c.Client.Do(req)
-}
-
-func (c *Client) TestQueryWithBody(ctx context.Context, id QueryID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) {
- req, err := NewTestQueryRequestWithBody(c.Server, id, contentType, body)
- if err != nil {
- return nil, err
- }
- req = req.WithContext(ctx)
- if err := c.applyEditors(ctx, req, reqEditors); err != nil {
- return nil, err
- }
- return c.Client.Do(req)
-}
-
-func (c *Client) TestQuery(ctx context.Context, id QueryID, body TestQueryJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) {
- req, err := NewTestQueryRequest(c.Server, id, body)
- if err != nil {
- return nil, err
- }
- req = req.WithContext(ctx)
- if err := c.applyEditors(ctx, req, reqEditors); err != nil {
- return nil, err
- }
- return c.Client.Do(req)
-}
-
// NewListAdminUsersRequest generates requests for ListAdminUsers
func NewListAdminUsersRequest(server string, params *ListAdminUsersParams) (*http.Request, error) {
var err error
@@ -3951,201 +3720,6 @@ func NewLogoutRequest(server string) (*http.Request, error) {
return req, nil
}
-// NewListQueriesRequest generates requests for ListQueries
-func NewListQueriesRequest(server string) (*http.Request, error) {
- var err error
-
- serverURL, err := url.Parse(server)
- if err != nil {
- return nil, err
- }
-
- operationPath := fmt.Sprintf("/query")
- if operationPath[0] == '/' {
- operationPath = "." + operationPath
- }
-
- queryURL, err := serverURL.Parse(operationPath)
- if err != nil {
- return nil, err
- }
-
- req, err := http.NewRequest("GET", queryURL.String(), nil)
- if err != nil {
- return nil, err
- }
-
- return req, nil
-}
-
-// NewCreateQueryRequest calls the generic CreateQuery builder with application/json body
-func NewCreateQueryRequest(server string, body CreateQueryJSONRequestBody) (*http.Request, error) {
- var bodyReader io.Reader
- buf, err := json.Marshal(body)
- if err != nil {
- return nil, err
- }
- bodyReader = bytes.NewReader(buf)
- return NewCreateQueryRequestWithBody(server, "application/json", bodyReader)
-}
-
-// NewCreateQueryRequestWithBody generates requests for CreateQuery with any type of body
-func NewCreateQueryRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) {
- var err error
-
- serverURL, err := url.Parse(server)
- if err != nil {
- return nil, err
- }
-
- operationPath := fmt.Sprintf("/query")
- if operationPath[0] == '/' {
- operationPath = "." + operationPath
- }
-
- queryURL, err := serverURL.Parse(operationPath)
- if err != nil {
- return nil, err
- }
-
- req, err := http.NewRequest("POST", queryURL.String(), body)
- if err != nil {
- return nil, err
- }
-
- req.Header.Add("Content-Type", contentType)
-
- return req, nil
-}
-
-// NewGetQueryRequest generates requests for GetQuery
-func NewGetQueryRequest(server string, id QueryID) (*http.Request, error) {
- var err error
-
- var pathParam0 string
-
- pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id)
- if err != nil {
- return nil, err
- }
-
- serverURL, err := url.Parse(server)
- if err != nil {
- return nil, err
- }
-
- operationPath := fmt.Sprintf("/query/%s", pathParam0)
- if operationPath[0] == '/' {
- operationPath = "." + operationPath
- }
-
- queryURL, err := serverURL.Parse(operationPath)
- if err != nil {
- return nil, err
- }
-
- req, err := http.NewRequest("GET", queryURL.String(), nil)
- if err != nil {
- return nil, err
- }
-
- return req, nil
-}
-
-// NewUpdateQueryRequest calls the generic UpdateQuery builder with application/json body
-func NewUpdateQueryRequest(server string, id QueryID, body UpdateQueryJSONRequestBody) (*http.Request, error) {
- var bodyReader io.Reader
- buf, err := json.Marshal(body)
- if err != nil {
- return nil, err
- }
- bodyReader = bytes.NewReader(buf)
- return NewUpdateQueryRequestWithBody(server, id, "application/json", bodyReader)
-}
-
-// NewUpdateQueryRequestWithBody generates requests for UpdateQuery with any type of body
-func NewUpdateQueryRequestWithBody(server string, id QueryID, contentType string, body io.Reader) (*http.Request, error) {
- var err error
-
- var pathParam0 string
-
- pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id)
- if err != nil {
- return nil, err
- }
-
- serverURL, err := url.Parse(server)
- if err != nil {
- return nil, err
- }
-
- operationPath := fmt.Sprintf("/query/%s", pathParam0)
- if operationPath[0] == '/' {
- operationPath = "." + operationPath
- }
-
- queryURL, err := serverURL.Parse(operationPath)
- if err != nil {
- return nil, err
- }
-
- req, err := http.NewRequest("PATCH", queryURL.String(), body)
- if err != nil {
- return nil, err
- }
-
- req.Header.Add("Content-Type", contentType)
-
- return req, nil
-}
-
-// NewTestQueryRequest calls the generic TestQuery builder with application/json body
-func NewTestQueryRequest(server string, id QueryID, body TestQueryJSONRequestBody) (*http.Request, error) {
- var bodyReader io.Reader
- buf, err := json.Marshal(body)
- if err != nil {
- return nil, err
- }
- bodyReader = bytes.NewReader(buf)
- return NewTestQueryRequestWithBody(server, id, "application/json", bodyReader)
-}
-
-// NewTestQueryRequestWithBody generates requests for TestQuery with any type of body
-func NewTestQueryRequestWithBody(server string, id QueryID, contentType string, body io.Reader) (*http.Request, error) {
- var err error
-
- var pathParam0 string
-
- pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id)
- if err != nil {
- return nil, err
- }
-
- serverURL, err := url.Parse(server)
- if err != nil {
- return nil, err
- }
-
- operationPath := fmt.Sprintf("/query/%s/test", pathParam0)
- if operationPath[0] == '/' {
- operationPath = "." + operationPath
- }
-
- queryURL, err := serverURL.Parse(operationPath)
- if err != nil {
- return nil, err
- }
-
- req, err := http.NewRequest("POST", queryURL.String(), body)
- if err != nil {
- return nil, err
- }
-
- req.Header.Add("Content-Type", contentType)
-
- return req, nil
-}
-
func (c *Client) applyEditors(ctx context.Context, req *http.Request, additionalEditors []RequestEditorFn) error {
for _, r := range c.RequestEditors {
if err := r(ctx, req); err != nil {
@@ -4331,27 +3905,6 @@ type ClientWithResponsesInterface interface {
// LogoutWithResponse request
LogoutWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*LogoutResponse, error)
-
- // ListQueriesWithResponse request
- ListQueriesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*ListQueriesResponse, error)
-
- // CreateQueryWithBodyWithResponse request with any body
- CreateQueryWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateQueryResponse, error)
-
- CreateQueryWithResponse(ctx context.Context, body CreateQueryJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateQueryResponse, error)
-
- // GetQueryWithResponse request
- GetQueryWithResponse(ctx context.Context, id QueryID, reqEditors ...RequestEditorFn) (*GetQueryResponse, error)
-
- // UpdateQueryWithBodyWithResponse request with any body
- UpdateQueryWithBodyWithResponse(ctx context.Context, id QueryID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UpdateQueryResponse, error)
-
- UpdateQueryWithResponse(ctx context.Context, id QueryID, body UpdateQueryJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateQueryResponse, error)
-
- // TestQueryWithBodyWithResponse request with any body
- TestQueryWithBodyWithResponse(ctx context.Context, id QueryID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*TestQueryResponse, error)
-
- TestQueryWithResponse(ctx context.Context, id QueryID, body TestQueryJSONRequestBody, reqEditors ...RequestEditorFn) (*TestQueryResponse, error)
}
type ListAdminUsersResponse struct {
@@ -5450,135 +5003,6 @@ func (r LogoutResponse) StatusCode() int {
return 0
}
-type ListQueriesResponse struct {
- Body []byte
- HTTPResponse *http.Response
- JSON200 *ListQueries
- JSON400 *InvalidRequest
- JSON401 *Unauthorized
- JSON429 *TooManyRequests
- JSON500 *InternalError
-}
-
-// Status returns HTTPResponse.Status
-func (r ListQueriesResponse) Status() string {
- if r.HTTPResponse != nil {
- return r.HTTPResponse.Status
- }
- return http.StatusText(0)
-}
-
-// StatusCode returns HTTPResponse.StatusCode
-func (r ListQueriesResponse) StatusCode() int {
- if r.HTTPResponse != nil {
- return r.HTTPResponse.StatusCode
- }
- return 0
-}
-
-type CreateQueryResponse struct {
- Body []byte
- HTTPResponse *http.Response
- JSON201 *IdMessage
- JSON400 *InvalidRequest
- JSON401 *Unauthorized
- JSON429 *TooManyRequests
- JSON500 *InternalError
-}
-
-// Status returns HTTPResponse.Status
-func (r CreateQueryResponse) Status() string {
- if r.HTTPResponse != nil {
- return r.HTTPResponse.Status
- }
- return http.StatusText(0)
-}
-
-// StatusCode returns HTTPResponse.StatusCode
-func (r CreateQueryResponse) StatusCode() int {
- if r.HTTPResponse != nil {
- return r.HTTPResponse.StatusCode
- }
- return 0
-}
-
-type GetQueryResponse struct {
- Body []byte
- HTTPResponse *http.Response
- JSON200 *Query
- JSON400 *InvalidRequest
- JSON401 *Unauthorized
- JSON429 *TooManyRequests
- JSON500 *InternalError
-}
-
-// Status returns HTTPResponse.Status
-func (r GetQueryResponse) Status() string {
- if r.HTTPResponse != nil {
- return r.HTTPResponse.Status
- }
- return http.StatusText(0)
-}
-
-// StatusCode returns HTTPResponse.StatusCode
-func (r GetQueryResponse) StatusCode() int {
- if r.HTTPResponse != nil {
- return r.HTTPResponse.StatusCode
- }
- return 0
-}
-
-type UpdateQueryResponse struct {
- Body []byte
- HTTPResponse *http.Response
- JSON400 *InvalidRequest
- JSON401 *Unauthorized
- JSON429 *TooManyRequests
- JSON500 *InternalError
-}
-
-// Status returns HTTPResponse.Status
-func (r UpdateQueryResponse) Status() string {
- if r.HTTPResponse != nil {
- return r.HTTPResponse.Status
- }
- return http.StatusText(0)
-}
-
-// StatusCode returns HTTPResponse.StatusCode
-func (r UpdateQueryResponse) StatusCode() int {
- if r.HTTPResponse != nil {
- return r.HTTPResponse.StatusCode
- }
- return 0
-}
-
-type TestQueryResponse struct {
- Body []byte
- HTTPResponse *http.Response
- JSON200 *QueryTestResponse
- JSON400 *InvalidRequest
- JSON401 *Unauthorized
- JSON429 *TooManyRequests
- JSON500 *InternalError
-}
-
-// Status returns HTTPResponse.Status
-func (r TestQueryResponse) Status() string {
- if r.HTTPResponse != nil {
- return r.HTTPResponse.Status
- }
- return http.StatusText(0)
-}
-
-// StatusCode returns HTTPResponse.StatusCode
-func (r TestQueryResponse) StatusCode() int {
- if r.HTTPResponse != nil {
- return r.HTTPResponse.StatusCode
- }
- return 0
-}
-
// ListAdminUsersWithResponse request returning *ListAdminUsersResponse
func (c *ClientWithResponses) ListAdminUsersWithResponse(ctx context.Context, params *ListAdminUsersParams, reqEditors ...RequestEditorFn) (*ListAdminUsersResponse, error) {
rsp, err := c.ListAdminUsers(ctx, params, reqEditors...)
@@ -6028,75 +5452,6 @@ func (c *ClientWithResponses) LogoutWithResponse(ctx context.Context, reqEditors
return ParseLogoutResponse(rsp)
}
-// ListQueriesWithResponse request returning *ListQueriesResponse
-func (c *ClientWithResponses) ListQueriesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*ListQueriesResponse, error) {
- rsp, err := c.ListQueries(ctx, reqEditors...)
- if err != nil {
- return nil, err
- }
- return ParseListQueriesResponse(rsp)
-}
-
-// CreateQueryWithBodyWithResponse request with arbitrary body returning *CreateQueryResponse
-func (c *ClientWithResponses) CreateQueryWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateQueryResponse, error) {
- rsp, err := c.CreateQueryWithBody(ctx, contentType, body, reqEditors...)
- if err != nil {
- return nil, err
- }
- return ParseCreateQueryResponse(rsp)
-}
-
-func (c *ClientWithResponses) CreateQueryWithResponse(ctx context.Context, body CreateQueryJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateQueryResponse, error) {
- rsp, err := c.CreateQuery(ctx, body, reqEditors...)
- if err != nil {
- return nil, err
- }
- return ParseCreateQueryResponse(rsp)
-}
-
-// GetQueryWithResponse request returning *GetQueryResponse
-func (c *ClientWithResponses) GetQueryWithResponse(ctx context.Context, id QueryID, reqEditors ...RequestEditorFn) (*GetQueryResponse, error) {
- rsp, err := c.GetQuery(ctx, id, reqEditors...)
- if err != nil {
- return nil, err
- }
- return ParseGetQueryResponse(rsp)
-}
-
-// UpdateQueryWithBodyWithResponse request with arbitrary body returning *UpdateQueryResponse
-func (c *ClientWithResponses) UpdateQueryWithBodyWithResponse(ctx context.Context, id QueryID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UpdateQueryResponse, error) {
- rsp, err := c.UpdateQueryWithBody(ctx, id, contentType, body, reqEditors...)
- if err != nil {
- return nil, err
- }
- return ParseUpdateQueryResponse(rsp)
-}
-
-func (c *ClientWithResponses) UpdateQueryWithResponse(ctx context.Context, id QueryID, body UpdateQueryJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateQueryResponse, error) {
- rsp, err := c.UpdateQuery(ctx, id, body, reqEditors...)
- if err != nil {
- return nil, err
- }
- return ParseUpdateQueryResponse(rsp)
-}
-
-// TestQueryWithBodyWithResponse request with arbitrary body returning *TestQueryResponse
-func (c *ClientWithResponses) TestQueryWithBodyWithResponse(ctx context.Context, id QueryID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*TestQueryResponse, error) {
- rsp, err := c.TestQueryWithBody(ctx, id, contentType, body, reqEditors...)
- if err != nil {
- return nil, err
- }
- return ParseTestQueryResponse(rsp)
-}
-
-func (c *ClientWithResponses) TestQueryWithResponse(ctx context.Context, id QueryID, body TestQueryJSONRequestBody, reqEditors ...RequestEditorFn) (*TestQueryResponse, error) {
- rsp, err := c.TestQuery(ctx, id, body, reqEditors...)
- if err != nil {
- return nil, err
- }
- return ParseTestQueryResponse(rsp)
-}
-
// ParseListAdminUsersResponse parses an HTTP response from a ListAdminUsersWithResponse call
func ParseListAdminUsersResponse(rsp *http.Response) (*ListAdminUsersResponse, error) {
bodyBytes, err := io.ReadAll(rsp.Body)
@@ -8472,266 +7827,3 @@ func ParseLogoutResponse(rsp *http.Response) (*LogoutResponse, error) {
return response, nil
}
-
-// ParseListQueriesResponse parses an HTTP response from a ListQueriesWithResponse call
-func ParseListQueriesResponse(rsp *http.Response) (*ListQueriesResponse, error) {
- bodyBytes, err := io.ReadAll(rsp.Body)
- defer func() { _ = rsp.Body.Close() }()
- if err != nil {
- return nil, err
- }
-
- response := &ListQueriesResponse{
- Body: bodyBytes,
- HTTPResponse: rsp,
- }
-
- switch {
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200:
- var dest ListQueries
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON200 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400:
- var dest InvalidRequest
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON400 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401:
- var dest Unauthorized
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON401 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 429:
- var dest TooManyRequests
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON429 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500:
- var dest InternalError
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON500 = &dest
-
- }
-
- return response, nil
-}
-
-// ParseCreateQueryResponse parses an HTTP response from a CreateQueryWithResponse call
-func ParseCreateQueryResponse(rsp *http.Response) (*CreateQueryResponse, error) {
- bodyBytes, err := io.ReadAll(rsp.Body)
- defer func() { _ = rsp.Body.Close() }()
- if err != nil {
- return nil, err
- }
-
- response := &CreateQueryResponse{
- Body: bodyBytes,
- HTTPResponse: rsp,
- }
-
- switch {
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 201:
- var dest IdMessage
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON201 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400:
- var dest InvalidRequest
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON400 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401:
- var dest Unauthorized
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON401 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 429:
- var dest TooManyRequests
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON429 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500:
- var dest InternalError
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON500 = &dest
-
- }
-
- return response, nil
-}
-
-// ParseGetQueryResponse parses an HTTP response from a GetQueryWithResponse call
-func ParseGetQueryResponse(rsp *http.Response) (*GetQueryResponse, error) {
- bodyBytes, err := io.ReadAll(rsp.Body)
- defer func() { _ = rsp.Body.Close() }()
- if err != nil {
- return nil, err
- }
-
- response := &GetQueryResponse{
- Body: bodyBytes,
- HTTPResponse: rsp,
- }
-
- switch {
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200:
- var dest Query
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON200 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400:
- var dest InvalidRequest
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON400 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401:
- var dest Unauthorized
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON401 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 429:
- var dest TooManyRequests
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON429 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500:
- var dest InternalError
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON500 = &dest
-
- }
-
- return response, nil
-}
-
-// ParseUpdateQueryResponse parses an HTTP response from a UpdateQueryWithResponse call
-func ParseUpdateQueryResponse(rsp *http.Response) (*UpdateQueryResponse, error) {
- bodyBytes, err := io.ReadAll(rsp.Body)
- defer func() { _ = rsp.Body.Close() }()
- if err != nil {
- return nil, err
- }
-
- response := &UpdateQueryResponse{
- Body: bodyBytes,
- HTTPResponse: rsp,
- }
-
- switch {
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400:
- var dest InvalidRequest
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON400 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401:
- var dest Unauthorized
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON401 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 429:
- var dest TooManyRequests
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON429 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500:
- var dest InternalError
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON500 = &dest
-
- }
-
- return response, nil
-}
-
-// ParseTestQueryResponse parses an HTTP response from a TestQueryWithResponse call
-func ParseTestQueryResponse(rsp *http.Response) (*TestQueryResponse, error) {
- bodyBytes, err := io.ReadAll(rsp.Body)
- defer func() { _ = rsp.Body.Close() }()
- if err != nil {
- return nil, err
- }
-
- response := &TestQueryResponse{
- Body: bodyBytes,
- HTTPResponse: rsp,
- }
-
- switch {
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200:
- var dest QueryTestResponse
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON200 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400:
- var dest InvalidRequest
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON400 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401:
- var dest Unauthorized
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON401 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 429:
- var dest TooManyRequests
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON429 = &dest
-
- case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500:
- var dest InternalError
- if err := json.Unmarshal(bodyBytes, &dest); err != nil {
- return nil, err
- }
- response.JSON500 = &dest
-
- }
-
- return response, nil
-}
diff --git a/queue_flow.md b/queue_flow.md
index 979f84d6..2ea91a25 100644
--- a/queue_flow.md
+++ b/queue_flow.md
@@ -5,7 +5,7 @@
This document describes the SQS queue-based processing pipeline that orchestrates document processing through various stages. The system uses AWS SQS queues to connect microservices (runners) in a sequential processing pipeline.
## Visual Pipeline Flow
-[mermaid link](https://mermaid.live/edit#pako:eNqdlWtv2zYUhv8KwaDABiiZJOqOoUDjdFg_dGvktAVmFwVDHV0QmdQoqY1n57-PF8nxXBcYKoCASL08z7mR2mEmCsAZLlvxldVUDujues2Rel68QEuCXn8BPqClGCUDu74kq5_Uh-uRPcDw67385eWNYONGy953raDFz5_Q5eXL_WHzH2JoyobRoRF8j5a33pofCG9pw9G7poO24YBuRxihR5QXKB85B9lPyFtv1Q9CwmfQFg3UaA0J5d7OfDU4u9FIssRNvCdrIveMUwsJdFCIQjAkgQlZaI_8VTGF8LnhzbH92VWlsSh_p6RvlOiE488c33JqYA-oGLtWBw690d1v0es7WqGa9rXGkmdsv-XsLJZYLNHYpRKdYMmMNbr9B9o2heIh1jZzohjlxjwqW1ppbPCMZS1QfpYbWG6guQutOgEHMziw8WoJ6iYBE20LTJUE9YMqJpVFr8HhM3iAx_NpDi031Nw7JTrBhjPW6PavPi6RVknKrLk_Fzma5lO3Rau_R5Db72c4ssRoZ3RnchzPzMj2NauhGFtAWt9MpS1VrHNwmhpb6llgbIGxBZ7AkhlmVPvfVfomlDVWQAe8AM4U2YCODtPCFB3pENBv6jzbD6_evfFWt9qAfj2A3E9T4cyesdNtow0mK9s73ybMep9Y75OdlZ3JVzqHYKT7azqwGnVSMOhtsmjbTg2qc2bCIEdhWF8_qNOvSnguGmPWqp79Tqc6f7H7vud-at1PbfInyLdBpO4chNmwv5NNVYFEtCxVY0Nh29zEYPxPjvxfDlt1m1V2zlra9zdQ6hKOgMqmbbOLsixi13X6QYoHyC4IIdP75demGOrM7x4ddYaEzC5c1z0xJI2jk6UkZgD3P2hJX5q0mp1Kk_I-TX7QFO2aQ2zUjen_N3NkSN_zjrpp1SBqBGqEakRqxGokji6fyePxptxzct_JiZMHTh46eeTksZMnjqqcTdV_CGSO-njVNJUKwa5hB29AbmhTqP_iTq-t8VDDBtY4U6_qOntY4zV_Ujo6DkI3D84GOYKDpRirGmclbXs1s81509BK0s0s6Sj_S4jDFIpG-fPW_oTNv9jBldToyaI-7XIhRj7gzPdiz1jA2Q4_4ozEwVVCUt91Yz8KQ8938BZnl0mcXBEv8RMSxF7ok-DJwf8YpneVem4QpUkUpcSPI-_pXxutlHc)
+
```mermaid
flowchart TB
%% S3 Event Source
@@ -25,46 +25,34 @@ flowchart TB
R4 -->|Clean per
collector standards| SQ5[document_text
Queue]
SQ5 --> R5{docTextRunner
:8085}
- R5 -->|AWS Textract
OCR extraction| SQ6[query_sync
Queue]
-
- SQ6 --> R6{querySyncRunner
:8087}
- R6 -->|Schedule queries
for document| SQ7[query
Queue]
-
- SQ7 --> R7{queryRunner
:8088}
- R7 -->|Handle query
dependencies| SQ7
+ R5 -->|AWS Textract
OCR extraction| DB[(PostgreSQL)]
%% Client Sync Flow
API1[Query API
:8080] -->|Client update| SQ8[client_sync
Queue]
SQ8 --> R8{clientSyncRunner
:8089}
R8 -->|Batch process
all client docs| SQ3
- %% Query Version Sync Flow
- API1 -->|Query update| SQ9[query_version_sync
Queue]
- SQ9 --> R9{queryVersionSyncRunner
:8090}
- R9 -->|Trigger affected
clients| SQ8
-
%% Styling
classDef queue fill:#ffd700,stroke:#333,stroke-width:2px,color:#000
classDef runner fill:#87ceeb,stroke:#333,stroke-width:2px,color:#000
classDef storage fill:#98fb98,stroke:#333,stroke-width:2px,color:#000
classDef api fill:#ffa07a,stroke:#333,stroke-width:2px,color:#000
- class SQ1,SQ2,SQ3,SQ4,SQ5,SQ6,SQ7,SQ8,SQ9 queue
- class R1,R2,R3,R4,R5,R6,R7,R8,R9 runner
- class S3 storage
+ class SQ1,SQ2,SQ3,SQ4,SQ5,SQ8 queue
+ class R1,R2,R3,R4,R5,R8 runner
+ class S3,DB storage
class API1 api
```
### Flow Legend
- 🟨 **Yellow boxes**: SQS Queues
- 🟦 **Blue diamonds**: Runner Services (with port numbers)
-- 🟩 **Green cylinder**: S3 Storage
+- 🟩 **Green cylinder**: S3 Storage / PostgreSQL Database
- 🟧 **Orange box**: REST API endpoint
- **Solid arrows**: Message flow direction
-- **Self-loop on queryRunner**: Handles query dependencies by re-queueing
## Detailed Process Flow
-[mermaid link](https://mermaid.live/edit#pako:eNp9lW9v2jAQxr-K5VfblFaEtCvkRaU1MAlp0xoSadKEVBnnAI_EoY4zlSK--84OgQYSkJAc--fnnjv_21GeJ0B9OpMFvJYgOYwEWyqWzSTB34YpLbjYMKlJ5LX0jQkrSKFzBS_wD7AnLKGENnB6JMcGnJZSgroERxPDJTkvM6RehBSdmqPJ9MBOkOoUjBqCxVbyTsGoFoyQ6hQMGoI8BSY7FYNaMTBYp2TckNTw1p10XCvGSHUJhjZpXFC1vZpxWGVswWs5h0eqU-kk1Jnlk0Ge80IvFUThj5msmMi7eXyMxj42iN0b5NOv-V_gOlDANCT-l88HcGzBqY8iaUoyKAq2hHpsioOjJ58EK-BrIhZEZJtcabSgVw1mgpFAJqaGRMhFTnbzkq9BO2QNW4esWLHa195GEzujNSR2N0Im5SYVHB2T-daqXGA2n-M6EwU8V0mDig7edjU0GZ28RJZo9xI1vfBUmAicSbv-DShoiWHqBalYinkKx3iBpdvjBRflZimml2yJPRDn1DOoRa6yalDIZWM8vpZ0bIl2E_GFCXt08K8Yx51zwmqRgKHIt98RiQ_QuVJkbqiTghX8yITXViiMLNFqNjyu0HeB09nGbhYstz00AoqPWHgZxKkOl1krrCQBxlfHsGZGe9CzHQobVMVb_hSuBsZvwEtdmdmejVU1UVCUqT7Nq2xO4ebV3Aim-B_liWniQlOHZqAyJhJ8ZXZm8ozqFWQwoz42E6bWM3x99sixUufmEqK-ViU4VOXlckX9BUsL_Co3CR6ew-t07MWL5U-eZ_UUSAR6_Vm9afZpc-hSmdgHSeNPBXkpNfX77p0VoP6OvlHf7bm3D3f9njcY9j13eN936Nb09m693sD1BgPPux_2eoO9Q99tyN7tYOC6_Ydhf-h-7WPjYf8f7gRf3A)
+
```mermaid
sequenceDiagram
participant S3
@@ -78,10 +66,6 @@ sequenceDiagram
participant DCR as docCleanRunner
participant DT as document_text Queue
participant DTR as docTextRunner
- participant QS as query_sync Queue
- participant QSR as querySyncRunner
- participant Q as query Queue
- participant QR as queryRunner
participant DB as PostgreSQL
S3->>SE: S3 Event (ObjectCreated:*)
@@ -107,17 +91,6 @@ sequenceDiagram
DTR->>DB: Check if text extracted
DTR-->>DTR: Call AWS Textract
DTR->>DB: Store extracted text
- DTR->>QS: Send {documentID}
-
- QS->>QSR: Poll message
- QSR->>DB: Find applicable queries
- QSR->>Q: Send {documentID, queryID} for each
-
- Q->>QR: Poll message
- QR->>DB: Check dependencies
- QR->>DB: Execute query
- QR->>DB: Store results
- QR-->>Q: Re-queue if dependencies pending
```
## S3 Event Trigger Configuration
@@ -171,9 +144,7 @@ aws s3api put-bucket-notification-configuration \
| **2** | `docInitRunner` | `document_init` | `document_sync` | Creates document records, checks for duplicates based on ETag hash |
| **3** | `docSyncRunner` | `document_sync` | `document_clean` | Validates client eligibility for document processing (can_sync flag) |
| **4** | `docCleanRunner` | `document_clean` | `document_text` | Cleans document content according to collector standards |
-| **5** | `docTextRunner` | `document_text` | `query_sync` | Extracts text using AWS Textract OCR service |
-| **6** | `querySyncRunner` | `query_sync` | `query` | Syncs and schedules queries that need to run for documents |
-| **7** | `queryRunner` | `query` | `query` (self-referential) | Executes queries with dependency chains, can re-queue for dependencies |
+| **5** | `docTextRunner` | `document_text` | - | Extracts text using AWS Textract OCR service, stores in database |
### Additional Async Runners
@@ -182,7 +153,6 @@ These runners are triggered by API calls or other events outside the main pipeli
| Runner Service | Reads From Queue | Writes To Queue | Trigger |
|---------------|------------------|-----------------|---------|
| `clientSyncRunner` | `client_sync` | `document_sync` | Triggered by API calls or when client configuration changes |
-| `queryVersionSyncRunner` | `query_version_sync` | `client_sync` | Triggered when query versions are updated via API |
## Pipeline Flow Diagrams
@@ -210,13 +180,7 @@ docCleanRunner
↓
docTextRunner
↓
-[query_sync queue]
- ↓
-querySyncRunner
- ↓
-[query queue]
- ↓
-queryRunner
+PostgreSQL (text stored)
```
### 2. Client Sync Flow (Batch Processing)
@@ -230,21 +194,6 @@ clientSyncRunner
[document_sync queue] → (feeds into main pipeline)
```
-### 3. Query Version Update Flow
-```
-Query Update (API)
- ↓
-[query_version_sync queue]
- ↓
-queryVersionSyncRunner
- ↓
-[client_sync queue]
- ↓
-clientSyncRunner
- ↓
-[document_sync queue] → (feeds into main pipeline)
-```
-
## Queue Names and Environment Variables
| Queue Purpose | Queue Name Variable | Queue URL Variable |
@@ -254,19 +203,15 @@ clientSyncRunner
| Document Sync | `QNAME_DOCUMENT_SYNC` | `DOCUMENT_SYNC_URL` |
| Document Clean | `QNAME_DOCUMENT_CLEAN` | `DOCUMENT_CLEAN_URL` |
| Document Text | `QNAME_DOCUMENT_TEXT` | `DOCUMENT_TEXT_URL` |
-| Query Sync | `QNAME_QUERY_SYNC` | `QUERY_SYNC_URL` |
-| Query Runner | `QNAME_QUERY_RUNNER` | `QUERY_URL` |
| Client Sync | `QNAME_CLIENT_SYNC` | `CLIENT_SYNC_URL` |
-| Query Version Sync | `QNAME_QUERY_VERSION_SYNC` | `QUERY_VERSION_SYNC_URL` |
## Key Design Principles
1. **Single Responsibility**: Each runner has one specific task in the pipeline
2. **Sequential Processing**: Documents flow through defined stages in order
3. **Idempotent Operations**: Duplicate detection prevents reprocessing
-4. **Async Reprocessing**: Client/query updates trigger document reprocessing
-5. **Self-Referential Queuing**: Query runner can re-queue items for dependency handling
-6. **Scalability**: Each runner can be scaled independently based on workload
+4. **Async Reprocessing**: Client updates trigger document reprocessing
+5. **Scalability**: Each runner can be scaled independently based on workload
## IAM Permissions Required
@@ -293,7 +238,4 @@ Runner ports for local development:
- `docSyncRunner`: 8083
- `docCleanRunner`: 8084
- `docTextRunner`: 8085
-- `querySyncRunner`: 8087
-- `queryRunner`: 8088
- `clientSyncRunner`: 8089
-- `queryVersionSyncRunner`: 8090
\ No newline at end of file
diff --git a/scripts/local-deployments.yml b/scripts/local-deployments.yml
index ea95174a..431fb23b 100644
--- a/scripts/local-deployments.yml
+++ b/scripts/local-deployments.yml
@@ -31,6 +31,13 @@ tasks:
COMPOSE_FILE: "{{.COMPOSE_FILE | default .LOCAL_COMPOSE_FILE}}"
cmds:
- docker compose -f {{.COMPOSE_FILE}} up --no-recreate --wait -d
+ up:infra:
+ desc: "Start only infrastructure containers (localstack, db, prometheus)"
+ internal: true
+ vars:
+ COMPOSE_FILE: "{{.COMPOSE_FILE | default .LOCAL_COMPOSE_FILE}}"
+ cmds:
+ - docker compose -f {{.COMPOSE_FILE}} up localstack db prometheus -d --wait
up:test:
cmds:
- task: up:cmd
@@ -55,8 +62,9 @@ tasks:
- task deps:tidy
- task docker:build
- task: build
- - task: up:cmd
+ - task: up:infra
- task: init
+ - task: up:cmd
refresh:
cmds:
- task deps:tidy
@@ -64,8 +72,9 @@ tasks:
- task docker:build:debug
- task: build
- task: down
- - task: up:cmd
+ - task: up:infra
- task: init
+ - task: up:cmd
init:
internal: true
cmds:
diff --git a/serviceAPIs/queryAPI.yaml b/serviceAPIs/queryAPI.yaml
index 13e44328..09fb2c95 100644
--- a/serviceAPIs/queryAPI.yaml
+++ b/serviceAPIs/queryAPI.yaml
@@ -27,8 +27,6 @@ tags:
description: Operations related to document labels and workflow tracking
- name: FieldExtractionService
description: Operations related to document field extractions
- - name: QueryService
- description: Operations related to queries
- name: ExportService
description: Operations related to exports
- name: AuthService
@@ -160,162 +158,7 @@ paths:
"500":
$ref: "#/components/responses/InternalError"
- /query:
- get:
- operationId: listQueries
- tags:
- - QueryService
- summary: List queries
- description: Retrieves a list of queries based on filter criteria.
- security:
- - jwtAuth: []
- responses:
- "200":
- description: A list of queries.
- headers:
- RateLimit:
- $ref: "#/components/headers/RateLimit"
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/ListQueries"
- "400":
- $ref: "#/components/responses/InvalidRequest"
- "401":
- $ref: "#/components/responses/Unauthorized"
- "429":
- $ref: "#/components/responses/TooManyRequests"
- "500":
- $ref: "#/components/responses/InternalError"
- post:
- operationId: createQuery
- tags:
- - QueryService
- summary: Create a new query
- description: Creates a new query with the provided details.
- security:
- - jwtAuth: []
- requestBody:
- description: The details required to create a query
- required: true
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/QueryCreate"
- responses:
- "201":
- description: Query created successfully.
- headers:
- RateLimit:
- $ref: "#/components/headers/RateLimit"
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/IdMessage"
- "400":
- $ref: "#/components/responses/InvalidRequest"
- "401":
- $ref: "#/components/responses/Unauthorized"
- "429":
- $ref: "#/components/responses/TooManyRequests"
- "500":
- $ref: "#/components/responses/InternalError"
-
- /query/{id}:
- parameters:
- - $ref: "#/components/parameters/QueryID"
- get:
- operationId: getQuery
- tags:
- - QueryService
- summary: Get a query by ID
- description: Retrieves a specific query by its ID.
- security:
- - jwtAuth: []
- responses:
- "200":
- description: Query details.
- headers:
- RateLimit:
- $ref: "#/components/headers/RateLimit"
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/Query"
- "400":
- $ref: "#/components/responses/InvalidRequest"
- "401":
- $ref: "#/components/responses/Unauthorized"
- "429":
- $ref: "#/components/responses/TooManyRequests"
- "500":
- $ref: "#/components/responses/InternalError"
- patch:
- operationId: updateQuery
- tags:
- - QueryService
- summary: Update a query
- description: Updates an existing query with new details.
- security:
- - jwtAuth: []
- requestBody:
- description: The update values for the desired query.
- required: true
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/QueryUpdate"
- responses:
- "200":
- description: Query updated successfully.
- headers:
- RateLimit:
- $ref: "#/components/headers/RateLimit"
- "400":
- $ref: "#/components/responses/InvalidRequest"
- "401":
- $ref: "#/components/responses/Unauthorized"
- "429":
- $ref: "#/components/responses/TooManyRequests"
- "500":
- $ref: "#/components/responses/InternalError"
-
- /query/{id}/test:
- parameters:
- - $ref: "#/components/parameters/QueryID"
- post:
- operationId: testQuery
- tags:
- - QueryService
- summary: Test a query
- description: Executes a test run of a query with the provided parameters.
- security:
- - jwtAuth: []
- requestBody:
- description: The query test requirements.
- required: true
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/QueryTestRequest"
- responses:
- "200":
- description: Test result.
- headers:
- RateLimit:
- $ref: "#/components/headers/RateLimit"
- content:
- application/json:
- schema:
- $ref: "#/components/schemas/QueryTestResponse"
- "400":
- $ref: "#/components/responses/InvalidRequest"
- "401":
- $ref: "#/components/responses/Unauthorized"
- "429":
- $ref: "#/components/responses/TooManyRequests"
- "500":
- $ref: "#/components/responses/InternalError"
+ # Query paths have been removed - see remove_query_plan.md
/client/{id}/status:
parameters:
@@ -1912,13 +1755,7 @@ components:
$ref: "#/components/schemas/ExportID"
description: The export ID.
- QueryID:
- in: path
- name: id
- required: true
- schema:
- $ref: "#/components/schemas/QueryID"
- description: The ID of the query.
+ # QueryID parameter has been removed - see remove_query_plan.md
DocumentID:
in: path
@@ -2078,13 +1915,6 @@ components:
maxLength: 36
pattern: "^.+$"
- QueryID:
- type: string
- format: uuid
- description: The query id.
- example: 019580df-ef65-7676-8de9-94435a93337a
- maxLength: 36
-
DocumentID:
type: string
format: uuid
@@ -2255,109 +2085,6 @@ components:
type: boolean
description: If the client is allowing active syncs
- QueryType:
- type: string
- enum:
- - JSON_EXTRACTOR
- - CONTEXT_FULL
- description: Specifies the type of the query.
-
- QueryConfig:
- type: string
- description: Configuration for the query.
- maxLength: 2048
- pattern: '^\{.*\}$'
-
- Query:
- description: A logic unit of execution.
- type: object
- properties:
- id:
- $ref: "#/components/schemas/QueryID"
- type:
- $ref: "#/components/schemas/QueryType"
- active_version:
- $ref: "#/components/schemas/Version"
- latest_version:
- $ref: "#/components/schemas/Version"
- config:
- $ref: "#/components/schemas/QueryConfig"
- required_queries:
- $ref: "#/components/schemas/RequiredQueryIDs"
- required:
- - id
- - type
- - active_version
- - latest_version
-
- ListQueries:
- description: A set of queries.
- type: object
- properties:
- queries:
- type: array
- maxItems: 512
- items:
- $ref: "#/components/schemas/Query"
- description: List of queries.
- required:
- - queries
-
- QueryCreate:
- description: The parameters required to create a query.
- type: object
- properties:
- type:
- $ref: "#/components/schemas/QueryType"
- config:
- $ref: "#/components/schemas/QueryConfig"
- required_queries:
- $ref: "#/components/schemas/RequiredQueryIDs"
- required:
- - type
-
- RequiredQueryIDs:
- type: array
- maxItems: 256
- items:
- $ref: "#/components/schemas/QueryID"
- description: List of required query IDs.
-
- QueryUpdate:
- description: The properties that may be updated for a query.
- type: object
- properties:
- config:
- $ref: "#/components/schemas/QueryConfig"
- active_version:
- $ref: "#/components/schemas/Version"
- required_queries:
- $ref: "#/components/schemas/RequiredQueryIDs"
-
- QueryTestRequest:
- description: The properties for a query test request.
- type: object
- properties:
- document_id:
- $ref: "#/components/schemas/DocumentID"
- query_version:
- $ref: "#/components/schemas/Version"
- required:
- - document_id
- - query_version
-
- QueryTestResponse:
- description: The response from a query test.
- type: object
- properties:
- value:
- type: string
- description: Result of the query test.
- pattern: "^.+$"
- maxLength: 2048
- required:
- - value
-
DocClient:
description: The properties of a client.
type: object
@@ -2451,7 +2178,6 @@ components:
- id
- client_id
- hash
- - fields
- hasTextRecord
- labels
properties:
@@ -2461,13 +2187,6 @@ components:
$ref: "#/components/schemas/ClientID"
hash:
$ref: "#/components/schemas/Hash"
- fields:
- type: object
- description: "The fields and the value for the document"
- example:
- property1: "string value"
- property2: 42
- property3: true
hasTextRecord:
type: boolean
description: "Whether a text extraction (field extraction) record exists for this document"
@@ -2603,24 +2322,14 @@ components:
$ref: "#/components/schemas/CodeVersion"
minimum_text_version:
$ref: "#/components/schemas/CodeVersion"
- fields:
- $ref: "#/components/schemas/CollectorFields"
description: Collector model.
required:
- client_id
- - fields
- minimum_cleaner_version
- minimum_text_version
- latest_version
- active_version
- CollectorFields:
- type: array
- maxItems: 256
- description: The fields in the collector.
- items:
- $ref: "#/components/schemas/CollectorField"
-
CollectorSet:
type: object
properties:
@@ -2630,8 +2339,6 @@ components:
$ref: "#/components/schemas/CodeVersion"
active_version:
$ref: "#/components/schemas/Version"
- fields:
- $ref: "#/components/schemas/CollectorFields"
description: Payload for updating a Collector.
CollectorFieldName:
@@ -2640,18 +2347,6 @@ components:
maxLength: 256
description: The output field name.
- CollectorField:
- type: object
- description: The field properties for the collector.
- properties:
- name:
- $ref: "#/components/schemas/CollectorFieldName"
- query_id:
- $ref: "#/components/schemas/QueryID"
- required:
- - name
- - query_id
-
ExportTrigger:
type: object
properties:
diff --git a/test/README.md b/test/README.md
new file mode 100644
index 00000000..f91829a4
--- /dev/null
+++ b/test/README.md
@@ -0,0 +1,78 @@
+# Integration Tests
+
+This directory contains two integration test implementations that test the same core API workflow but serve different purposes.
+
+## Test Files
+
+### `text_extraction_integration_test.go` - Automated CI/CD Testing
+
+**Purpose**: Automated integration testing for CI/CD pipelines.
+
+**How it works**:
+- Self-contained using testcontainers (spins up PostgreSQL, S3 mock, and API server automatically)
+- Uses the generated Go API client for type-safe requests
+- Runs as part of `task test:functional` or `task fullsuite:ci`
+
+**When to use**:
+- Local development verification before committing
+- CI/CD pipeline testing
+- Automated regression testing
+
+**Run with**:
+```bash
+go test -v ./test/... -run TestE2EDocWorkflow
+# Or via task
+task test:functional
+```
+
+---
+
+### `text_extraction_integration_test.sh` - Manual Testing
+
+**Purpose**: Manual end-to-end testing against a running API server.
+
+**How it works**:
+- Uses curl/jq to make REST API calls
+- Requires an external API server already running (auth bypass mode)
+- Creates comprehensive test data with realistic healthcare contract values
+- Demonstrates API usage for UI developers
+
+**When to use**:
+- Testing against local docker-compose stack
+- Testing against AWS dev/UAT deployments
+- Populating a development environment with sample data
+- Demonstrating API endpoints to UI developers
+
+**Run with**:
+```bash
+# Against local stack
+./test/text_extraction_integration_test.sh http://localhost:8080
+
+# Against AWS deployment
+./test/text_extraction_integration_test.sh http://your-alb-url.us-east-2.elb.amazonaws.com
+```
+
+**Requirements**:
+- bash 4.0+
+- curl
+- jq
+- zip
+- Target API must be running in auth bypass mode
+
+---
+
+## Test Coverage
+
+Both tests exercise the same core workflow:
+
+1. **Client Setup**: Create, verify, list, update client
+2. **Document Upload**: Batch ZIP upload, single document upload
+3. **Label Operations**: Apply labels, verify, query by label
+4. **Folder Metrics**: List folders, get metrics per folder
+5. **Field Extractions**: Create, version history, retrieve specific versions
+
+The Go test includes additional scenarios:
+- Deep folder hierarchy (12 levels)
+- Large array field extraction (100 items)
+- Label reapplication history
+- Cross-feature integration (folders + labels + field extractions)
diff --git a/test/process_test.go b/test/process_test.go
deleted file mode 100644
index 8378e725..00000000
--- a/test/process_test.go
+++ /dev/null
@@ -1,362 +0,0 @@
-package endtoend_test
-
-import (
- "testing"
-
- "queryorchestration/internal/serviceconfig"
- "queryorchestration/internal/serviceconfig/aws"
- "queryorchestration/internal/serviceconfig/objectstore"
- "queryorchestration/internal/test"
-
- queryapitest "queryorchestration/internal/test/queryAPI"
- queryapi "queryorchestration/pkg/queryAPI"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- "golang.org/x/sync/errgroup"
-)
-
-type Config struct {
- serviceconfig.BaseConfig
- aws.AWSConfig
- objectstore.ObjectStoreConfig
-}
-
-func TestProcess(t *testing.T) {
- t.Run("basic upload", func(t *testing.T) {
- cfg := &Config{}
-
- net, clean := test.CreateFullNetwork(t, t.Context(), cfg)
- defer clean()
-
- var clientId queryapi.ClientID
- var jsonId queryapi.QueryID
- var textractExpectation test.MockExpectation
- textractBody := "Hello World"
-
- // Use errgroup to properly propagate errors and avoid indefinite hangs
- // If any goroutine fails, g.Wait() returns the error immediately
- g := new(errgroup.Group)
-
- g.Go(func() error {
- _, jsonId = queryapitest.CreateDependentQueries(t, net.Client)
- return nil
- })
- g.Go(func() error {
- clientId = queryapitest.CreateClientWithSync(t, net.Client)
- return nil
- })
-
- if err := g.Wait(); err != nil {
- t.Fatalf("setup phase 1 failed: %v", err)
- }
-
- g = new(errgroup.Group)
-
- g.Go(func() error {
- textractExpectation = test.CreateDetectDocumentTextExpectation(
- t,
- net.Dependencies.MockServer,
- textractBody,
- )
- return nil
- })
- g.Go(func() error {
- queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId)
- return nil
- })
- g.Go(func() error {
- queryapitest.CreateFile(t, net.Client, queryapitest.File{
- Filename: "helloworld.pdf",
- Content: []byte(pdfHelloWorld),
- ClientID: clientId,
- })
- return nil
- })
-
- if err := g.Wait(); err != nil {
- t.Fatalf("setup phase 2 failed: %v", err)
- }
-
- test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request)
- queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
-
- docs, err := net.Client.ListDocumentsByClientIdWithResponse(t.Context(), clientId)
- require.NoError(t, err)
- assert.Len(t, *docs.JSON200, 1)
- doc := (*docs.JSON200)[0]
-
- docRes, err := net.Client.GetDocumentWithResponse(t.Context(), doc.Id, nil)
- require.NoError(t, err)
- require.NotNil(t, docRes)
- require.NotNil(t, docRes.JSON200)
-
- // Verify core document fields
- assert.Equal(t, doc.Id, docRes.JSON200.Id)
- assert.Equal(t, doc.Hash, docRes.JSON200.Hash)
- assert.Equal(t, clientId, docRes.JSON200.ClientId)
- assert.Equal(t, map[string]any{"JSON_QUERY": "valueone"}, docRes.JSON200.Fields)
- assert.Equal(t, []queryapi.LabelRecord{}, docRes.JSON200.Labels)
-
- // Verify enriched fields are populated (values depend on upload)
- assert.NotNil(t, docRes.JSON200.Filename, "Filename should be populated")
- assert.Equal(t, "helloworld.pdf", *docRes.JSON200.Filename)
- assert.NotNil(t, docRes.JSON200.OriginalPath, "OriginalPath should be populated")
-
- // Test with textRecord=true parameter
- includeTextRecord := true
- docResWithText, err := net.Client.GetDocumentWithResponse(t.Context(), doc.Id, &queryapi.GetDocumentParams{
- TextRecord: &includeTextRecord,
- })
- require.NoError(t, err)
- require.NotNil(t, docResWithText.JSON200)
- // Document doesn't have a field extraction in this test, so hasTextRecord should be false
- assert.False(t, docResWithText.JSON200.HasTextRecord)
- assert.Nil(t, docResWithText.JSON200.TextRecord)
- })
- t.Run("update config", func(t *testing.T) {
- cfg := &Config{}
-
- net, clean := test.CreateFullNetwork(t, t.Context(), cfg)
- defer clean()
-
- var clientId queryapi.ClientID
- var jsonId queryapi.QueryID
- var textractExpectation test.MockExpectation
- textractBody := "Hello World"
-
- g := new(errgroup.Group)
-
- g.Go(func() error {
- _, jsonId = queryapitest.CreateDependentQueries(t, net.Client)
- return nil
- })
- g.Go(func() error {
- clientId = queryapitest.CreateClientWithSync(t, net.Client)
- return nil
- })
-
- if err := g.Wait(); err != nil {
- t.Fatalf("setup phase 1 failed: %v", err)
- }
-
- g = new(errgroup.Group)
-
- g.Go(func() error {
- queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId)
- return nil
- })
- g.Go(func() error {
- queryapitest.CreateFile(t, net.Client, queryapitest.File{
- Filename: "helloworld.pdf",
- Content: []byte(pdfHelloWorld),
- ClientID: clientId,
- })
- return nil
- })
- g.Go(func() error {
- textractExpectation = test.CreateDetectDocumentTextExpectation(
- t,
- net.Dependencies.MockServer,
- textractBody,
- )
- return nil
- })
-
- if err := g.Wait(); err != nil {
- t.Fatalf("setup phase 2 failed: %v", err)
- }
-
- test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request)
- queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
-
- jcfg := `{"path":"keytwo"}`
- av := int32(2)
- _, err := net.Client.UpdateQueryWithResponse(t.Context(), jsonId, queryapi.QueryUpdate{
- ActiveVersion: &av,
- Config: &jcfg,
- })
- require.NoError(t, err)
-
- queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
-
- docs, err := net.Client.ListDocumentsByClientIdWithResponse(t.Context(), clientId)
- require.NoError(t, err)
- assert.Len(t, *docs.JSON200, 1)
- doc := (*docs.JSON200)[0]
-
- docRes, err := net.Client.GetDocumentWithResponse(t.Context(), doc.Id, nil)
- require.NoError(t, err)
- require.NotNil(t, docRes)
- require.NotNil(t, docRes.JSON200)
-
- // Verify core document fields
- assert.Equal(t, doc.Id, docRes.JSON200.Id)
- assert.Equal(t, doc.Hash, docRes.JSON200.Hash)
- assert.Equal(t, clientId, docRes.JSON200.ClientId)
- assert.Equal(t, map[string]any{"JSON_QUERY": "valuetwo"}, docRes.JSON200.Fields)
- assert.Equal(t, []queryapi.LabelRecord{}, docRes.JSON200.Labels)
-
- // Verify enriched fields are populated (values depend on upload)
- assert.NotNil(t, docRes.JSON200.Filename, "Filename should be populated")
- assert.Equal(t, "helloworld.pdf", *docRes.JSON200.Filename)
- assert.NotNil(t, docRes.JSON200.OriginalPath, "OriginalPath should be populated")
- })
- t.Run("test multiple versions", func(t *testing.T) {
- cfg := &Config{}
-
- net, clean := test.CreateFullNetwork(t, t.Context(), cfg)
- defer clean()
-
- var clientId queryapi.ClientID
- var jsonId queryapi.QueryID
- var textractExpectation test.MockExpectation
- textractBody := "Hello World"
-
- g := new(errgroup.Group)
-
- g.Go(func() error {
- _, jsonId = queryapitest.CreateDependentQueries(t, net.Client)
-
- jcfg := `{"path":"keytwo"}`
- _, err := net.Client.UpdateQueryWithResponse(t.Context(), jsonId, queryapi.QueryUpdate{
- Config: &jcfg,
- })
- require.NoError(t, err)
- return nil
- })
- g.Go(func() error {
- clientId = queryapitest.CreateClientWithSync(t, net.Client)
- return nil
- })
-
- if err := g.Wait(); err != nil {
- t.Fatalf("setup phase 1 failed: %v", err)
- }
-
- g = new(errgroup.Group)
-
- g.Go(func() error {
- textractExpectation = test.CreateDetectDocumentTextExpectation(
- t,
- net.Dependencies.MockServer,
- textractBody,
- )
- return nil
- })
- g.Go(func() error {
- queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId)
- return nil
- })
- g.Go(func() error {
- queryapitest.CreateFile(t, net.Client, queryapitest.File{
- Filename: "helloworld.pdf",
- Content: []byte(pdfHelloWorld),
- ClientID: clientId,
- })
- return nil
- })
-
- if err := g.Wait(); err != nil {
- t.Fatalf("setup phase 2 failed: %v", err)
- }
-
- test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request)
- queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
-
- docs, err := net.Client.ListDocumentsByClientIdWithResponse(t.Context(), clientId)
- require.NoError(t, err)
- assert.Len(t, *docs.JSON200, 1)
- doc := (*docs.JSON200)[0]
-
- g = new(errgroup.Group)
-
- // Capture doc.Id to avoid closure issues
- docId := doc.Id
- g.Go(func() error {
- testRes, err := net.Client.TestQueryWithResponse(t.Context(), jsonId, queryapi.QueryTestRequest{
- QueryVersion: 1,
- DocumentId: docId,
- })
- require.NoError(t, err)
- assert.Equal(t, "valueone", testRes.JSON200.Value)
- return nil
- })
-
- g.Go(func() error {
- testRes, err := net.Client.TestQueryWithResponse(t.Context(), jsonId, queryapi.QueryTestRequest{
- QueryVersion: 2,
- DocumentId: docId,
- })
- require.NoError(t, err)
- assert.Equal(t, "valuetwo", testRes.JSON200.Value)
- return nil
- })
-
- if err := g.Wait(); err != nil {
- t.Fatalf("test phase failed: %v", err)
- }
- })
-}
-
-const pdfHelloWorld = `%PDF-1.4
-%����
-1 0 obj
-<<
- /Type /Catalog
- /Pages 2 0 R
- /Version /1.4
->>
-endobj
-2 0 obj
-<<
- /Type /Pages
- /Kids [3 0 R]
- /Count 1
->>
-endobj
-3 0 obj
-<<
- /Type /Page
- /Parent 2 0 R
- /MediaBox [0 0 612 792]
- /Resources <<
- /Font <<
- /F1 <<
- /Type /Font
- /Subtype /Type1
- /BaseFont /Helvetica
- >>
- >>
- >>
- /Contents 4 0 R
->>
-endobj
-4 0 obj
-<<
- /Length
- 44
->>
-stream
-BT
-/F1 24 Tf
-100 700 Td
-(Hello World!) Tj
-ET
-endstream
-endobj
-xref
-0 5
-0000000000 65535 f
-0000000015 00000 n
-0000000086 00000 n
-0000000151 00000 n
-0000000376 00000 n
-trailer
-<<
- /Size 5
- /Root 1 0 R
->>
-startxref
-472
-%%EOF`
diff --git a/test/queryAPI/collectorservice_test.go b/test/queryAPI/collectorservice_test.go
index fe26dead..cf40ed9b 100644
--- a/test/queryAPI/collectorservice_test.go
+++ b/test/queryAPI/collectorservice_test.go
@@ -14,7 +14,6 @@ import (
"github.com/stretchr/testify/require"
- "github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
)
@@ -25,6 +24,8 @@ type Config struct {
objectstore.ObjectStoreConfig
}
+// TestCollectorService tests the collector API endpoints.
+// Note: Query functionality has been removed. See remove_query_plan.md for details.
func TestCollectorService(t *testing.T) {
ctx := t.Context()
@@ -36,21 +37,6 @@ func TestCollectorService(t *testing.T) {
client, err := queryapi.NewClientWithResponses(c.API.URI)
require.NoError(t, err)
- contextRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
- Type: queryapi.CONTEXTFULL,
- })
- require.NoError(t, err)
-
- jsoncfg := "{\"path\":\"key\"}"
- jsonRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
- Type: queryapi.JSONEXTRACTOR,
- Config: &jsoncfg,
- RequiredQueries: &[]types.UUID{
- contextRes.JSON201.Id,
- },
- })
- require.NoError(t, err)
-
clientRes, err := client.CreateClientWithResponse(ctx, queryapi.ClientCreate{
Name: "example_name",
Id: "ID",
@@ -66,21 +52,13 @@ func TestCollectorService(t *testing.T) {
assert.Equal(t, int32(0), collRes.JSON200.LatestVersion)
assert.Equal(t, int64(0), collRes.JSON200.MinimumCleanerVersion)
assert.Equal(t, int64(0), collRes.JSON200.MinimumTextVersion)
- assert.Len(t, collRes.JSON200.Fields, 0)
av := int32(1)
- fields := []queryapi.CollectorField{
- {
- Name: "json",
- QueryId: jsonRes.JSON201.Id,
- },
- }
minVersion := int64(1000)
uRes, err := client.SetCollectorByClientIdWithResponse(ctx, id, queryapi.CollectorSet{
ActiveVersion: &av,
MinimumCleanerVersion: &minVersion,
MinimumTextVersion: &minVersion,
- Fields: &fields,
})
require.NoError(t, err)
test.AssertStatus(t, http.StatusOK, uRes.HTTPResponse)
@@ -93,8 +71,6 @@ func TestCollectorService(t *testing.T) {
assert.Equal(t, int32(1), collRes.JSON200.LatestVersion)
assert.Equal(t, minVersion, collRes.JSON200.MinimumCleanerVersion)
assert.Equal(t, minVersion, collRes.JSON200.MinimumTextVersion)
- assert.Len(t, collRes.JSON200.Fields, 1)
- assert.ElementsMatch(t, fields, collRes.JSON200.Fields)
test.AssertMessageBody(t, cfg, c.Dependencies.QueueURLs[test.ClientSyncRunnerName], regexp.MustCompile(`{"id":".+"}`))
}
diff --git a/test/queryAPI/queryservice_test.go b/test/queryAPI/queryservice_test.go
deleted file mode 100644
index 9eba1a8b..00000000
--- a/test/queryAPI/queryservice_test.go
+++ /dev/null
@@ -1,160 +0,0 @@
-package endtoend_test
-
-import (
- "regexp"
- "testing"
-
- "queryorchestration/internal/test"
- queryapi "queryorchestration/pkg/queryAPI"
-
- "github.com/stretchr/testify/require"
-
- "github.com/oapi-codegen/runtime/types"
- "github.com/stretchr/testify/assert"
-)
-
-func TestQueryAPI(t *testing.T) {
- t.Run("list no queries", func(t *testing.T) {
- cfg := &Config{}
-
- c, cleanup := test.CreateAPINetworkWithParams(t, cfg, test.QueryAPI, &test.FullDependenciesParams{
- NoObjectStore: true,
- Runners: &[]test.RunnerName{},
- })
- defer cleanup()
-
- client, err := queryapi.NewClientWithResponses(c.API.URI)
- require.NoError(t, err)
-
- queriesRes, err := client.ListQueriesWithResponse(t.Context())
- require.NoError(t, err)
- assert.Len(t, queriesRes.JSON200.Queries, 0)
- })
- t.Run("create and get query", func(t *testing.T) {
- ctx := t.Context()
-
- cfg := &Config{}
-
- c, cleanup := test.CreateAPINetworkWithParams(t, cfg, test.QueryAPI, &test.FullDependenciesParams{
- NoObjectStore: true,
- Runners: &[]test.RunnerName{},
- })
- defer cleanup()
-
- client, err := queryapi.NewClientWithResponses(c.API.URI)
- require.NoError(t, err)
-
- jcfg := `{"path": "createkey"}`
- idRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
- Type: queryapi.JSONEXTRACTOR,
- Config: &jcfg,
- })
- require.NoError(t, err)
- assert.NotNil(t, idRes)
- assert.NotNil(t, idRes.JSON201)
- jsonID := idRes.JSON201.Id
-
- queryRes, err := client.GetQueryWithResponse(ctx, jsonID)
- require.NoError(t, err)
- assert.Equal(t, jsonID, queryRes.JSON200.Id)
- assert.Equal(t, queryapi.JSONEXTRACTOR, queryRes.JSON200.Type)
- assert.Equal(t, int32(1), queryRes.JSON200.ActiveVersion)
- assert.Equal(t, int32(1), queryRes.JSON200.LatestVersion)
- assert.Equal(t, jcfg, *queryRes.JSON200.Config)
- assert.Nil(t, queryRes.JSON200.RequiredQueries)
-
- queriesRes, err := client.ListQueriesWithResponse(ctx)
- require.NoError(t, err)
- assert.Len(t, queriesRes.JSON200.Queries, 1)
- assert.Equal(t, jcfg, *queriesRes.JSON200.Queries[0].Config)
- })
- t.Run("update query", func(t *testing.T) {
- ctx := t.Context()
-
- cfg := &Config{}
-
- c, cleanup := test.CreateAPINetworkWithParams(t, cfg, test.QueryAPI, &test.FullDependenciesParams{
- NoObjectStore: true,
- Runners: &[]test.RunnerName{test.QueryVersionSyncRunnerName},
- })
- defer cleanup()
-
- client, err := queryapi.NewClientWithResponses(c.API.URI)
- require.NoError(t, err)
-
- jcfg := "{}"
- idRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
- Type: queryapi.JSONEXTRACTOR,
- Config: &jcfg,
- })
- require.NoError(t, err)
- assert.NotNil(t, idRes)
- assert.NotNil(t, idRes.JSON201)
- jsonID := idRes.JSON201.Id
-
- aV := int32(2)
- newJcfg := `{"path": "keyone"}`
- res, err := client.UpdateQueryWithResponse(ctx, jsonID, queryapi.QueryUpdate{
- ActiveVersion: &aV,
- Config: &newJcfg,
- })
- require.NoError(t, err)
- assert.NotNil(t, res)
-
- test.AssertMessageBody(t, cfg, c.Dependencies.QueueURLs[test.QueryVersionSyncRunnerName], regexp.MustCompile(`{"id":".+"}`))
-
- queryRes, err := client.GetQueryWithResponse(ctx, jsonID)
- require.NoError(t, err)
- assert.Equal(t, jsonID, queryRes.JSON200.Id)
- assert.Equal(t, queryapi.JSONEXTRACTOR, queryRes.JSON200.Type)
- assert.Equal(t, int32(2), queryRes.JSON200.ActiveVersion)
- assert.Equal(t, int32(2), queryRes.JSON200.LatestVersion)
- assert.Equal(t, newJcfg, *queryRes.JSON200.Config)
- assert.Nil(t, queryRes.JSON200.RequiredQueries)
- })
- t.Run("multiple dependent queries", func(t *testing.T) {
- ctx := t.Context()
-
- cfg := &Config{}
-
- c, cleanup := test.CreateAPINetworkWithParams(t, cfg, test.QueryAPI, &test.FullDependenciesParams{
- NoObjectStore: true,
- Runners: &[]test.RunnerName{},
- })
- defer cleanup()
-
- client, err := queryapi.NewClientWithResponses(c.API.URI)
- require.NoError(t, err)
-
- idRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
- Type: queryapi.CONTEXTFULL,
- })
- require.NoError(t, err)
- assert.NotNil(t, idRes)
- assert.NotNil(t, idRes.JSON201)
- contextID := idRes.JSON201.Id
- assert.NotEmpty(t, contextID)
-
- jcfg := "{}"
- idRes, err = client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
- Type: queryapi.JSONEXTRACTOR,
- Config: &jcfg,
- RequiredQueries: &[]types.UUID{
- contextID,
- },
- })
- require.NoError(t, err)
- assert.NotNil(t, idRes)
- assert.NotNil(t, idRes.JSON201)
- jsonID := idRes.JSON201.Id
-
- queryRes, err := client.GetQueryWithResponse(ctx, jsonID)
- require.NoError(t, err)
- assert.Equal(t, jsonID, queryRes.JSON200.Id)
- assert.Equal(t, queryapi.JSONEXTRACTOR, queryRes.JSON200.Type)
- assert.Equal(t, int32(1), queryRes.JSON200.ActiveVersion)
- assert.Equal(t, int32(1), queryRes.JSON200.LatestVersion)
- assert.Equal(t, jcfg, *queryRes.JSON200.Config)
- assert.ElementsMatch(t, []types.UUID{contextID}, *queryRes.JSON200.RequiredQueries)
- })
-}
diff --git a/vendor/github.com/tidwall/gjson/LICENSE b/vendor/github.com/tidwall/gjson/LICENSE
deleted file mode 100644
index 58f5819a..00000000
--- a/vendor/github.com/tidwall/gjson/LICENSE
+++ /dev/null
@@ -1,20 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2016 Josh Baker
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md
deleted file mode 100644
index 387766d5..00000000
--- a/vendor/github.com/tidwall/gjson/README.md
+++ /dev/null
@@ -1,492 +0,0 @@
-
-
-
-
- 
-
-
-
-
-
-
-
-
-get json values quickly
-
-GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document.
-It has features such as [one line retrieval](#get-a-value), [dot notation paths](#path-syntax), [iteration](#iterate-through-an-object-or-array), and [parsing json lines](#json-lines).
-
-Also check out [SJSON](https://github.com/tidwall/sjson) for modifying json, and the [JJ](https://github.com/tidwall/jj) command line tool.
-
-This README is a quick overview of how to use GJSON, for more information check out [GJSON Syntax](SYNTAX.md).
-
-GJSON is also available for [Python](https://github.com/volans-/gjson-py) and [Rust](https://github.com/tidwall/gjson.rs)
-
-Getting Started
-===============
-
-## Installing
-
-To start using GJSON, install Go and run `go get`:
-
-```sh
-$ go get -u github.com/tidwall/gjson
-```
-
-This will retrieve the library.
-
-## Get a value
-Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". When the value is found it's returned immediately.
-
-```go
-package main
-
-import "github.com/tidwall/gjson"
-
-const json = `{"name":{"first":"Janet","last":"Prichard"},"age":47}`
-
-func main() {
- value := gjson.Get(json, "name.last")
- println(value.String())
-}
-```
-
-This will print:
-
-```
-Prichard
-```
-*There's also the [GetMany](#get-multiple-values-at-once) function to get multiple values at once, and [GetBytes](#working-with-bytes) for working with JSON byte slices.*
-
-## Path Syntax
-
-Below is a quick overview of the path syntax, for more complete information please
-check out [GJSON Syntax](SYNTAX.md).
-
-A path is a series of keys separated by a dot.
-A key may contain special wildcard characters '\*' and '?'.
-To access an array value use the index as the key.
-To get the number of elements in an array or to access a child path, use the '#' character.
-The dot and wildcard characters can be escaped with '\\'.
-
-```json
-{
- "name": {"first": "Tom", "last": "Anderson"},
- "age":37,
- "children": ["Sara","Alex","Jack"],
- "fav.movie": "Deer Hunter",
- "friends": [
- {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]},
- {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]},
- {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]}
- ]
-}
-```
-```
-"name.last" >> "Anderson"
-"age" >> 37
-"children" >> ["Sara","Alex","Jack"]
-"children.#" >> 3
-"children.1" >> "Alex"
-"child*.2" >> "Jack"
-"c?ildren.0" >> "Sara"
-"fav\.movie" >> "Deer Hunter"
-"friends.#.first" >> ["Dale","Roger","Jane"]
-"friends.1.last" >> "Craig"
-```
-
-You can also query an array for the first match by using `#(...)`, or find all
-matches with `#(...)#`. Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=`
-comparison operators and the simple pattern matching `%` (like) and `!%`
-(not like) operators.
-
-```
-friends.#(last=="Murphy").first >> "Dale"
-friends.#(last=="Murphy")#.first >> ["Dale","Jane"]
-friends.#(age>45)#.last >> ["Craig","Murphy"]
-friends.#(first%"D*").last >> "Murphy"
-friends.#(first!%"D*").last >> "Craig"
-friends.#(nets.#(=="fb"))#.first >> ["Dale","Roger"]
-```
-
-*Please note that prior to v1.3.0, queries used the `#[...]` brackets. This was
-changed in v1.3.0 as to avoid confusion with the new
-[multipath](SYNTAX.md#multipaths) syntax. For backwards compatibility,
-`#[...]` will continue to work until the next major release.*
-
-## Result Type
-
-GJSON supports the json types `string`, `number`, `bool`, and `null`.
-Arrays and Objects are returned as their raw json types.
-
-The `Result` type holds one of these:
-
-```
-bool, for JSON booleans
-float64, for JSON numbers
-string, for JSON string literals
-nil, for JSON null
-```
-
-To directly access the value:
-
-```go
-result.Type // can be String, Number, True, False, Null, or JSON
-result.Str // holds the string
-result.Num // holds the float64 number
-result.Raw // holds the raw json
-result.Index // index of raw value in original json, zero means index unknown
-result.Indexes // indexes of all the elements that match on a path containing the '#' query character.
-```
-
-There are a variety of handy functions that work on a result:
-
-```go
-result.Exists() bool
-result.Value() interface{}
-result.Int() int64
-result.Uint() uint64
-result.Float() float64
-result.String() string
-result.Bool() bool
-result.Time() time.Time
-result.Array() []gjson.Result
-result.Map() map[string]gjson.Result
-result.Get(path string) Result
-result.ForEach(iterator func(key, value Result) bool)
-result.Less(token Result, caseSensitive bool) bool
-```
-
-The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types:
-
-```go
-boolean >> bool
-number >> float64
-string >> string
-null >> nil
-array >> []interface{}
-object >> map[string]interface{}
-```
-
-The `result.Array()` function returns back an array of values.
-If the result represents a non-existent value, then an empty array will be returned.
-If the result is not a JSON array, the return value will be an array containing one result.
-
-### 64-bit integers
-
-The `result.Int()` and `result.Uint()` calls are capable of reading all 64 bits, allowing for large JSON integers.
-
-```go
-result.Int() int64 // -9223372036854775808 to 9223372036854775807
-result.Uint() uint64 // 0 to 18446744073709551615
-```
-
-## Modifiers and path chaining
-
-New in version 1.2 is support for modifier functions and path chaining.
-
-A modifier is a path component that performs custom processing on the
-json.
-
-Multiple paths can be "chained" together using the pipe character.
-This is useful for getting results from a modified query.
-
-For example, using the built-in `@reverse` modifier on the above json document,
-we'll get `children` array and reverse the order:
-
-```
-"children|@reverse" >> ["Jack","Alex","Sara"]
-"children|@reverse|0" >> "Jack"
-```
-
-There are currently the following built-in modifiers:
-
-- `@reverse`: Reverse an array or the members of an object.
-- `@ugly`: Remove all whitespace from a json document.
-- `@pretty`: Make the json document more human readable.
-- `@this`: Returns the current element. It can be used to retrieve the root element.
-- `@valid`: Ensure the json document is valid.
-- `@flatten`: Flattens an array.
-- `@join`: Joins multiple objects into a single object.
-- `@keys`: Returns an array of keys for an object.
-- `@values`: Returns an array of values for an object.
-- `@tostr`: Converts json to a string. Wraps a json string.
-- `@fromstr`: Converts a string from json. Unwraps a json string.
-- `@group`: Groups arrays of objects. See [e4fc67c](https://github.com/tidwall/gjson/commit/e4fc67c92aeebf2089fabc7872f010e340d105db).
-- `@dig`: Search for a value without providing its entire path. See [e8e87f2](https://github.com/tidwall/gjson/commit/e8e87f2a00dc41f3aba5631094e21f59a8cf8cbf).
-
-### Modifier arguments
-
-A modifier may accept an optional argument. The argument can be a valid JSON
-document or just characters.
-
-For example, the `@pretty` modifier takes a json object as its argument.
-
-```
-@pretty:{"sortKeys":true}
-```
-
-Which makes the json pretty and orders all of its keys.
-
-```json
-{
- "age":37,
- "children": ["Sara","Alex","Jack"],
- "fav.movie": "Deer Hunter",
- "friends": [
- {"age": 44, "first": "Dale", "last": "Murphy"},
- {"age": 68, "first": "Roger", "last": "Craig"},
- {"age": 47, "first": "Jane", "last": "Murphy"}
- ],
- "name": {"first": "Tom", "last": "Anderson"}
-}
-```
-
-*The full list of `@pretty` options are `sortKeys`, `indent`, `prefix`, and `width`.
-Please see [Pretty Options](https://github.com/tidwall/pretty#customized-output) for more information.*
-
-### Custom modifiers
-
-You can also add custom modifiers.
-
-For example, here we create a modifier that makes the entire json document upper
-or lower case.
-
-```go
-gjson.AddModifier("case", func(json, arg string) string {
- if arg == "upper" {
- return strings.ToUpper(json)
- }
- if arg == "lower" {
- return strings.ToLower(json)
- }
- return json
-})
-```
-
-```
-"children|@case:upper" >> ["SARA","ALEX","JACK"]
-"children|@case:lower|@reverse" >> ["jack","alex","sara"]
-```
-
-## JSON Lines
-
-There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array.
-
-For example:
-
-```
-{"name": "Gilbert", "age": 61}
-{"name": "Alexa", "age": 34}
-{"name": "May", "age": 57}
-{"name": "Deloise", "age": 44}
-```
-
-```
-..# >> 4
-..1 >> {"name": "Alexa", "age": 34}
-..3 >> {"name": "Deloise", "age": 44}
-..#.name >> ["Gilbert","Alexa","May","Deloise"]
-..#(name="May").age >> 57
-```
-
-The `ForEachLines` function will iterate through JSON lines.
-
-```go
-gjson.ForEachLine(json, func(line gjson.Result) bool{
- println(line.String())
- return true
-})
-```
-
-## Get nested array values
-
-Suppose you want all the last names from the following json:
-
-```json
-{
- "programmers": [
- {
- "firstName": "Janet",
- "lastName": "McLaughlin",
- }, {
- "firstName": "Elliotte",
- "lastName": "Hunter",
- }, {
- "firstName": "Jason",
- "lastName": "Harold",
- }
- ]
-}
-```
-
-You would use the path "programmers.#.lastName" like such:
-
-```go
-result := gjson.Get(json, "programmers.#.lastName")
-for _, name := range result.Array() {
- println(name.String())
-}
-```
-
-You can also query an object inside an array:
-
-```go
-name := gjson.Get(json, `programmers.#(lastName="Hunter").firstName`)
-println(name.String()) // prints "Elliotte"
-```
-
-## Iterate through an object or array
-
-The `ForEach` function allows for quickly iterating through an object or array.
-The key and value are passed to the iterator function for objects.
-Only the value is passed for arrays.
-Returning `false` from an iterator will stop iteration.
-
-```go
-result := gjson.Get(json, "programmers")
-result.ForEach(func(key, value gjson.Result) bool {
- println(value.String())
- return true // keep iterating
-})
-```
-
-## Simple Parse and Get
-
-There's a `Parse(json)` function that will do a simple parse, and `result.Get(path)` that will search a result.
-
-For example, all of these will return the same result:
-
-```go
-gjson.Parse(json).Get("name").Get("last")
-gjson.Get(json, "name").Get("last")
-gjson.Get(json, "name.last")
-```
-
-## Check for the existence of a value
-
-Sometimes you just want to know if a value exists.
-
-```go
-value := gjson.Get(json, "name.last")
-if !value.Exists() {
- println("no last name")
-} else {
- println(value.String())
-}
-
-// Or as one step
-if gjson.Get(json, "name.last").Exists() {
- println("has a last name")
-}
-```
-
-## Validate JSON
-
-The `Get*` and `Parse*` functions expects that the json is well-formed. Bad json will not panic, but it may return back unexpected results.
-
-If you are consuming JSON from an unpredictable source then you may want to validate prior to using GJSON.
-
-```go
-if !gjson.Valid(json) {
- return errors.New("invalid json")
-}
-value := gjson.Get(json, "name.last")
-```
-
-## Unmarshal to a map
-
-To unmarshal to a `map[string]interface{}`:
-
-```go
-m, ok := gjson.Parse(json).Value().(map[string]interface{})
-if !ok {
- // not a map
-}
-```
-
-## Working with Bytes
-
-If your JSON is contained in a `[]byte` slice, there's the [GetBytes](https://godoc.org/github.com/tidwall/gjson#GetBytes) function. This is preferred over `Get(string(data), path)`.
-
-```go
-var json []byte = ...
-result := gjson.GetBytes(json, path)
-```
-
-If you are using the `gjson.GetBytes(json, path)` function and you want to avoid converting `result.Raw` to a `[]byte`, then you can use this pattern:
-
-```go
-var json []byte = ...
-result := gjson.GetBytes(json, path)
-var raw []byte
-if result.Index > 0 {
- raw = json[result.Index:result.Index+len(result.Raw)]
-} else {
- raw = []byte(result.Raw)
-}
-```
-
-This is a best-effort no allocation sub slice of the original json. This method utilizes the `result.Index` field, which is the position of the raw data in the original json. It's possible that the value of `result.Index` equals zero, in which case the `result.Raw` is converted to a `[]byte`.
-
-## Performance
-
-Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/),
-[ffjson](https://github.com/pquerna/ffjson),
-[EasyJSON](https://github.com/mailru/easyjson),
-[jsonparser](https://github.com/buger/jsonparser),
-and [json-iterator](https://github.com/json-iterator/go)
-
-```
-BenchmarkGJSONGet-10 17893731 202.1 ns/op 0 B/op 0 allocs/op
-BenchmarkGJSONUnmarshalMap-10 1663548 2157 ns/op 1920 B/op 26 allocs/op
-BenchmarkJSONUnmarshalMap-10 832236 4279 ns/op 2920 B/op 68 allocs/op
-BenchmarkJSONUnmarshalStruct-10 1076475 3219 ns/op 920 B/op 12 allocs/op
-BenchmarkJSONDecoder-10 585729 6126 ns/op 3845 B/op 160 allocs/op
-BenchmarkFFJSONLexer-10 2508573 1391 ns/op 880 B/op 8 allocs/op
-BenchmarkEasyJSONLexer-10 3000000 537.9 ns/op 501 B/op 5 allocs/op
-BenchmarkJSONParserGet-10 13707510 263.9 ns/op 21 B/op 0 allocs/op
-BenchmarkJSONIterator-10 3000000 561.2 ns/op 693 B/op 14 allocs/op
-```
-
-JSON document used:
-
-```json
-{
- "widget": {
- "debug": "on",
- "window": {
- "title": "Sample Konfabulator Widget",
- "name": "main_window",
- "width": 500,
- "height": 500
- },
- "image": {
- "src": "Images/Sun.png",
- "hOffset": 250,
- "vOffset": 250,
- "alignment": "center"
- },
- "text": {
- "data": "Click Here",
- "size": 36,
- "style": "bold",
- "vOffset": 100,
- "alignment": "center",
- "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;"
- }
- }
-}
-```
-
-Each operation was rotated through one of the following search paths:
-
-```
-widget.window.name
-widget.image.hOffset
-widget.text.onMouseUp
-```
-
-**
-
-*These benchmarks were run on a MacBook Pro M1 Max using Go 1.22 and can be found [here](https://github.com/tidwall/gjson-benchmarks).*
diff --git a/vendor/github.com/tidwall/gjson/SYNTAX.md b/vendor/github.com/tidwall/gjson/SYNTAX.md
deleted file mode 100644
index a3f0fac2..00000000
--- a/vendor/github.com/tidwall/gjson/SYNTAX.md
+++ /dev/null
@@ -1,360 +0,0 @@
-# GJSON Path Syntax
-
-A GJSON Path is a text string syntax that describes a search pattern for quickly retrieving values from a JSON payload.
-
-This document is designed to explain the structure of a GJSON Path through examples.
-
-- [Path structure](#path-structure)
-- [Basic](#basic)
-- [Wildcards](#wildcards)
-- [Escape Character](#escape-character)
-- [Arrays](#arrays)
-- [Queries](#queries)
-- [Dot vs Pipe](#dot-vs-pipe)
-- [Modifiers](#modifiers)
-- [Multipaths](#multipaths)
-- [Literals](#literals)
-
-The definitive implementation is [github.com/tidwall/gjson](https://github.com/tidwall/gjson).
-Use the [GJSON Playground](https://gjson.dev) to experiment with the syntax online.
-
-## Path structure
-
-A GJSON Path is intended to be easily expressed as a series of components separated by a `.` character.
-
-Along with `.` character, there are a few more that have special meaning, including `|`, `#`, `@`, `\`, `*`, `!`, and `?`.
-
-## Example
-
-Given this JSON
-
-```json
-{
- "name": {"first": "Tom", "last": "Anderson"},
- "age":37,
- "children": ["Sara","Alex","Jack"],
- "fav.movie": "Deer Hunter",
- "friends": [
- {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]},
- {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]},
- {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]}
- ]
-}
-```
-
-The following GJSON Paths evaluate to the accompanying values.
-
-### Basic
-
-In many cases you'll just want to retrieve values by object name or array index.
-
-```go
-name.last "Anderson"
-name.first "Tom"
-age 37
-children ["Sara","Alex","Jack"]
-children.0 "Sara"
-children.1 "Alex"
-friends.1 {"first": "Roger", "last": "Craig", "age": 68}
-friends.1.first "Roger"
-```
-
-### Wildcards
-
-A key may contain the special wildcard characters `*` and `?`.
-The `*` will match on any zero+ characters, and `?` matches on any one character.
-
-```go
-child*.2 "Jack"
-c?ildren.0 "Sara"
-```
-
-### Escape character
-
-Special purpose characters, such as `.`, `*`, and `?` can be escaped with `\`.
-
-```go
-fav\.movie "Deer Hunter"
-```
-
-You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in your source code.
-
-```go
-// Go
-val := gjson.Get(json, "fav\\.movie") // must escape the slash
-val := gjson.Get(json, `fav\.movie`) // no need to escape the slash
-```
-
-```rust
-// Rust
-let val = gjson::get(json, "fav\\.movie") // must escape the slash
-let val = gjson::get(json, r#"fav\.movie"#) // no need to escape the slash
-```
-
-
-### Arrays
-
-The `#` character allows for digging into JSON Arrays.
-
-To get the length of an array you'll just use the `#` all by itself.
-
-```go
-friends.# 3
-friends.#.age [44,68,47]
-```
-
-### Queries
-
-You can also query an array for the first match by using `#(...)`, or find all matches with `#(...)#`.
-Queries support the `==`, `!=`, `<`, `<=`, `>`, `>=` comparison operators,
-and the simple pattern matching `%` (like) and `!%` (not like) operators.
-
-```go
-friends.#(last=="Murphy").first "Dale"
-friends.#(last=="Murphy")#.first ["Dale","Jane"]
-friends.#(age>45)#.last ["Craig","Murphy"]
-friends.#(first%"D*").last "Murphy"
-friends.#(first!%"D*").last "Craig"
-```
-
-To query for a non-object value in an array, you can forgo the string to the right of the operator.
-
-```go
-children.#(!%"*a*") "Alex"
-children.#(%"*a*")# ["Sara","Jack"]
-```
-
-Nested queries are allowed.
-
-```go
-friends.#(nets.#(=="fb"))#.first >> ["Dale","Roger"]
-```
-
-*Please note that prior to v1.3.0, queries used the `#[...]` brackets. This was
-changed in v1.3.0 as to avoid confusion with the new [multipath](#multipaths)
-syntax. For backwards compatibility, `#[...]` will continue to work until the
-next major release.*
-
-The `~` (tilde) operator will convert a value to a boolean before comparison.
-
-Supported tilde comparison type are:
-
-```
-~true Converts true-ish values to true
-~false Converts false-ish and non-existent values to true
-~null Converts null and non-existent values to true
-~* Converts any existing value to true
-```
-
-For example, using the following JSON:
-
-```json
-{
- "vals": [
- { "a": 1, "b": "data" },
- { "a": 2, "b": true },
- { "a": 3, "b": false },
- { "a": 4, "b": "0" },
- { "a": 5, "b": 0 },
- { "a": 6, "b": "1" },
- { "a": 7, "b": 1 },
- { "a": 8, "b": "true" },
- { "a": 9, "b": false },
- { "a": 10, "b": null },
- { "a": 11 }
- ]
-}
-```
-
-To query for all true-ish or false-ish values:
-
-```
-vals.#(b==~true)#.a >> [2,6,7,8]
-vals.#(b==~false)#.a >> [3,4,5,9,10,11]
-```
-
-The last value which was non-existent is treated as `false`
-
-To query for null and explicit value existence:
-
-```
-vals.#(b==~null)#.a >> [10,11]
-vals.#(b==~*)#.a >> [1,2,3,4,5,6,7,8,9,10]
-vals.#(b!=~*)#.a >> [11]
-```
-
-### Dot vs Pipe
-
-The `.` is standard separator, but it's also possible to use a `|`.
-In most cases they both end up returning the same results.
-The cases where`|` differs from `.` is when it's used after the `#` for [Arrays](#arrays) and [Queries](#queries).
-
-Here are some examples
-
-```go
-friends.0.first "Dale"
-friends|0.first "Dale"
-friends.0|first "Dale"
-friends|0|first "Dale"
-friends|# 3
-friends.# 3
-friends.#(last="Murphy")# [{"first": "Dale", "last": "Murphy", "age": 44},{"first": "Jane", "last": "Murphy", "age": 47}]
-friends.#(last="Murphy")#.first ["Dale","Jane"]
-friends.#(last="Murphy")#|first
-friends.#(last="Murphy")#.0 []
-friends.#(last="Murphy")#|0 {"first": "Dale", "last": "Murphy", "age": 44}
-friends.#(last="Murphy")#.# []
-friends.#(last="Murphy")#|# 2
-```
-
-Let's break down a few of these.
-
-The path `friends.#(last="Murphy")#` all by itself results in
-
-```json
-[{"first": "Dale", "last": "Murphy", "age": 44},{"first": "Jane", "last": "Murphy", "age": 47}]
-```
-
-The `.first` suffix will process the `first` path on each array element *before* returning the results. Which becomes
-
-```json
-["Dale","Jane"]
-```
-
-But the `|first` suffix actually processes the `first` path *after* the previous result.
-Since the previous result is an array, not an object, it's not possible to process
-because `first` does not exist.
-
-Yet, `|0` suffix returns
-
-```json
-{"first": "Dale", "last": "Murphy", "age": 44}
-```
-
-Because `0` is the first index of the previous result.
-
-### Modifiers
-
-A modifier is a path component that performs custom processing on the JSON.
-
-For example, using the built-in `@reverse` modifier on the above JSON payload will reverse the `children` array:
-
-```go
-children.@reverse ["Jack","Alex","Sara"]
-children.@reverse.0 "Jack"
-```
-
-There are currently the following built-in modifiers:
-
-- `@reverse`: Reverse an array or the members of an object.
-- `@ugly`: Remove all whitespace from JSON.
-- `@pretty`: Make the JSON more human readable.
-- `@this`: Returns the current element. It can be used to retrieve the root element.
-- `@valid`: Ensure the json document is valid.
-- `@flatten`: Flattens an array.
-- `@join`: Joins multiple objects into a single object.
-- `@keys`: Returns an array of keys for an object.
-- `@values`: Returns an array of values for an object.
-- `@tostr`: Converts json to a string. Wraps a json string.
-- `@fromstr`: Converts a string from json. Unwraps a json string.
-- `@group`: Groups arrays of objects. See [e4fc67c](https://github.com/tidwall/gjson/commit/e4fc67c92aeebf2089fabc7872f010e340d105db).
-- `@dig`: Search for a value without providing its entire path. See [e8e87f2](https://github.com/tidwall/gjson/commit/e8e87f2a00dc41f3aba5631094e21f59a8cf8cbf).
-
-#### Modifier arguments
-
-A modifier may accept an optional argument. The argument can be a valid JSON payload or just characters.
-
-For example, the `@pretty` modifier takes a json object as its argument.
-
-```
-@pretty:{"sortKeys":true}
-```
-
-Which makes the json pretty and orders all of its keys.
-
-```json
-{
- "age":37,
- "children": ["Sara","Alex","Jack"],
- "fav.movie": "Deer Hunter",
- "friends": [
- {"age": 44, "first": "Dale", "last": "Murphy"},
- {"age": 68, "first": "Roger", "last": "Craig"},
- {"age": 47, "first": "Jane", "last": "Murphy"}
- ],
- "name": {"first": "Tom", "last": "Anderson"}
-}
-```
-
-*The full list of `@pretty` options are `sortKeys`, `indent`, `prefix`, and `width`.
-Please see [Pretty Options](https://github.com/tidwall/pretty#customized-output) for more information.*
-
-#### Custom modifiers
-
-You can also add custom modifiers.
-
-For example, here we create a modifier which makes the entire JSON payload upper or lower case.
-
-```go
-gjson.AddModifier("case", func(json, arg string) string {
- if arg == "upper" {
- return strings.ToUpper(json)
- }
- if arg == "lower" {
- return strings.ToLower(json)
- }
- return json
-})
-"children.@case:upper" ["SARA","ALEX","JACK"]
-"children.@case:lower.@reverse" ["jack","alex","sara"]
-```
-
-*Note: Custom modifiers are not yet available in the Rust version*
-
-### Multipaths
-
-Starting with v1.3.0, GJSON added the ability to join multiple paths together
-to form new documents. Wrapping comma-separated paths between `[...]` or
-`{...}` will result in a new array or object, respectively.
-
-For example, using the given multipath:
-
-```
-{name.first,age,"the_murphys":friends.#(last="Murphy")#.first}
-```
-
-Here we selected the first name, age, and the first name for friends with the
-last name "Murphy".
-
-You'll notice that an optional key can be provided, in this case
-"the_murphys", to force assign a key to a value. Otherwise, the name of the
-actual field will be used, in this case "first". If a name cannot be
-determined, then "_" is used.
-
-This results in
-
-```json
-{"first":"Tom","age":37,"the_murphys":["Dale","Jane"]}
-```
-
-### Literals
-
-Starting with v1.12.0, GJSON added support of json literals, which provides a way for constructing static blocks of json. This is can be particularly useful when constructing a new json document using [multipaths](#multipaths).
-
-A json literal begins with the '!' declaration character.
-
-For example, using the given multipath:
-
-```
-{name.first,age,"company":!"Happysoft","employed":!true}
-```
-
-Here we selected the first name and age. Then add two new fields, "company" and "employed".
-
-This results in
-
-```json
-{"first":"Tom","age":37,"company":"Happysoft","employed":true}
-```
-
-*See issue [#249](https://github.com/tidwall/gjson/issues/249) for additional context on JSON Literals.*
diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go
deleted file mode 100644
index 5aa2a4ff..00000000
--- a/vendor/github.com/tidwall/gjson/gjson.go
+++ /dev/null
@@ -1,3603 +0,0 @@
-// Package gjson provides searching for json strings.
-package gjson
-
-import (
- "strconv"
- "strings"
- "time"
- "unicode/utf16"
- "unicode/utf8"
- "unsafe"
-
- "github.com/tidwall/match"
- "github.com/tidwall/pretty"
-)
-
-// Type is Result type
-type Type int
-
-const (
- // Null is a null json value
- Null Type = iota
- // False is a json false boolean
- False
- // Number is json number
- Number
- // String is a json string
- String
- // True is a json true boolean
- True
- // JSON is a raw block of JSON
- JSON
-)
-
-// String returns a string representation of the type.
-func (t Type) String() string {
- switch t {
- default:
- return ""
- case Null:
- return "Null"
- case False:
- return "False"
- case Number:
- return "Number"
- case String:
- return "String"
- case True:
- return "True"
- case JSON:
- return "JSON"
- }
-}
-
-// Result represents a json value that is returned from Get().
-type Result struct {
- // Type is the json type
- Type Type
- // Raw is the raw json
- Raw string
- // Str is the json string
- Str string
- // Num is the json number
- Num float64
- // Index of raw value in original json, zero means index unknown
- Index int
- // Indexes of all the elements that match on a path containing the '#'
- // query character.
- Indexes []int
-}
-
-// String returns a string representation of the value.
-func (t Result) String() string {
- switch t.Type {
- default:
- return ""
- case False:
- return "false"
- case Number:
- if len(t.Raw) == 0 {
- // calculated result
- return strconv.FormatFloat(t.Num, 'f', -1, 64)
- }
- var i int
- if t.Raw[0] == '-' {
- i++
- }
- for ; i < len(t.Raw); i++ {
- if t.Raw[i] < '0' || t.Raw[i] > '9' {
- return strconv.FormatFloat(t.Num, 'f', -1, 64)
- }
- }
- return t.Raw
- case String:
- return t.Str
- case JSON:
- return t.Raw
- case True:
- return "true"
- }
-}
-
-// Bool returns an boolean representation.
-func (t Result) Bool() bool {
- switch t.Type {
- default:
- return false
- case True:
- return true
- case String:
- b, _ := strconv.ParseBool(strings.ToLower(t.Str))
- return b
- case Number:
- return t.Num != 0
- }
-}
-
-// Int returns an integer representation.
-func (t Result) Int() int64 {
- switch t.Type {
- default:
- return 0
- case True:
- return 1
- case String:
- n, _ := parseInt(t.Str)
- return n
- case Number:
- // try to directly convert the float64 to int64
- i, ok := safeInt(t.Num)
- if ok {
- return i
- }
- // now try to parse the raw string
- i, ok = parseInt(t.Raw)
- if ok {
- return i
- }
- // fallback to a standard conversion
- return int64(t.Num)
- }
-}
-
-// Uint returns an unsigned integer representation.
-func (t Result) Uint() uint64 {
- switch t.Type {
- default:
- return 0
- case True:
- return 1
- case String:
- n, _ := parseUint(t.Str)
- return n
- case Number:
- // try to directly convert the float64 to uint64
- i, ok := safeInt(t.Num)
- if ok && i >= 0 {
- return uint64(i)
- }
- // now try to parse the raw string
- u, ok := parseUint(t.Raw)
- if ok {
- return u
- }
- // fallback to a standard conversion
- return uint64(t.Num)
- }
-}
-
-// Float returns an float64 representation.
-func (t Result) Float() float64 {
- switch t.Type {
- default:
- return 0
- case True:
- return 1
- case String:
- n, _ := strconv.ParseFloat(t.Str, 64)
- return n
- case Number:
- return t.Num
- }
-}
-
-// Time returns a time.Time representation.
-func (t Result) Time() time.Time {
- res, _ := time.Parse(time.RFC3339, t.String())
- return res
-}
-
-// Array returns back an array of values.
-// If the result represents a null value or is non-existent, then an empty
-// array will be returned.
-// If the result is not a JSON array, the return value will be an
-// array containing one result.
-func (t Result) Array() []Result {
- if t.Type == Null {
- return []Result{}
- }
- if !t.IsArray() {
- return []Result{t}
- }
- r := t.arrayOrMap('[', false)
- return r.a
-}
-
-// IsObject returns true if the result value is a JSON object.
-func (t Result) IsObject() bool {
- return t.Type == JSON && len(t.Raw) > 0 && t.Raw[0] == '{'
-}
-
-// IsArray returns true if the result value is a JSON array.
-func (t Result) IsArray() bool {
- return t.Type == JSON && len(t.Raw) > 0 && t.Raw[0] == '['
-}
-
-// IsBool returns true if the result value is a JSON boolean.
-func (t Result) IsBool() bool {
- return t.Type == True || t.Type == False
-}
-
-// ForEach iterates through values.
-// If the result represents a non-existent value, then no values will be
-// iterated. If the result is an Object, the iterator will pass the key and
-// value of each item. If the result is an Array, the iterator will only pass
-// the value of each item. If the result is not a JSON array or object, the
-// iterator will pass back one value equal to the result.
-func (t Result) ForEach(iterator func(key, value Result) bool) {
- if !t.Exists() {
- return
- }
- if t.Type != JSON {
- iterator(Result{}, t)
- return
- }
- json := t.Raw
- var obj bool
- var i int
- var key, value Result
- for ; i < len(json); i++ {
- if json[i] == '{' {
- i++
- key.Type = String
- obj = true
- break
- } else if json[i] == '[' {
- i++
- key.Type = Number
- key.Num = -1
- break
- }
- if json[i] > ' ' {
- return
- }
- }
- var str string
- var vesc bool
- var ok bool
- var idx int
- for ; i < len(json); i++ {
- if obj {
- if json[i] != '"' {
- continue
- }
- s := i
- i, str, vesc, ok = parseString(json, i+1)
- if !ok {
- return
- }
- if vesc {
- key.Str = unescape(str[1 : len(str)-1])
- } else {
- key.Str = str[1 : len(str)-1]
- }
- key.Raw = str
- key.Index = s + t.Index
- } else {
- key.Num += 1
- }
- for ; i < len(json); i++ {
- if json[i] <= ' ' || json[i] == ',' || json[i] == ':' {
- continue
- }
- break
- }
- s := i
- i, value, ok = parseAny(json, i, true)
- if !ok {
- return
- }
- if t.Indexes != nil {
- if idx < len(t.Indexes) {
- value.Index = t.Indexes[idx]
- }
- } else {
- value.Index = s + t.Index
- }
- if !iterator(key, value) {
- return
- }
- idx++
- }
-}
-
-// Map returns back a map of values. The result should be a JSON object.
-// If the result is not a JSON object, the return value will be an empty map.
-func (t Result) Map() map[string]Result {
- if t.Type != JSON {
- return map[string]Result{}
- }
- r := t.arrayOrMap('{', false)
- return r.o
-}
-
-// Get searches result for the specified path.
-// The result should be a JSON array or object.
-func (t Result) Get(path string) Result {
- r := Get(t.Raw, path)
- if r.Indexes != nil {
- for i := 0; i < len(r.Indexes); i++ {
- r.Indexes[i] += t.Index
- }
- } else {
- r.Index += t.Index
- }
- return r
-}
-
-type arrayOrMapResult struct {
- a []Result
- ai []interface{}
- o map[string]Result
- oi map[string]interface{}
- vc byte
-}
-
-func (t Result) arrayOrMap(vc byte, valueize bool) (r arrayOrMapResult) {
- var json = t.Raw
- var i int
- var value Result
- var count int
- var key Result
- if vc == 0 {
- for ; i < len(json); i++ {
- if json[i] == '{' || json[i] == '[' {
- r.vc = json[i]
- i++
- break
- }
- if json[i] > ' ' {
- goto end
- }
- }
- } else {
- for ; i < len(json); i++ {
- if json[i] == vc {
- i++
- break
- }
- if json[i] > ' ' {
- goto end
- }
- }
- r.vc = vc
- }
- if r.vc == '{' {
- if valueize {
- r.oi = make(map[string]interface{})
- } else {
- r.o = make(map[string]Result)
- }
- } else {
- if valueize {
- r.ai = make([]interface{}, 0)
- } else {
- r.a = make([]Result, 0)
- }
- }
- for ; i < len(json); i++ {
- if json[i] <= ' ' {
- continue
- }
- // get next value
- if json[i] == ']' || json[i] == '}' {
- break
- }
- switch json[i] {
- default:
- if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' {
- value.Type = Number
- value.Raw, value.Num = tonum(json[i:])
- value.Str = ""
- } else {
- continue
- }
- case '{', '[':
- value.Type = JSON
- value.Raw = squash(json[i:])
- value.Str, value.Num = "", 0
- case 'n':
- value.Type = Null
- value.Raw = tolit(json[i:])
- value.Str, value.Num = "", 0
- case 't':
- value.Type = True
- value.Raw = tolit(json[i:])
- value.Str, value.Num = "", 0
- case 'f':
- value.Type = False
- value.Raw = tolit(json[i:])
- value.Str, value.Num = "", 0
- case '"':
- value.Type = String
- value.Raw, value.Str = tostr(json[i:])
- value.Num = 0
- }
- value.Index = i + t.Index
-
- i += len(value.Raw) - 1
-
- if r.vc == '{' {
- if count%2 == 0 {
- key = value
- } else {
- if valueize {
- if _, ok := r.oi[key.Str]; !ok {
- r.oi[key.Str] = value.Value()
- }
- } else {
- if _, ok := r.o[key.Str]; !ok {
- r.o[key.Str] = value
- }
- }
- }
- count++
- } else {
- if valueize {
- r.ai = append(r.ai, value.Value())
- } else {
- r.a = append(r.a, value)
- }
- }
- }
-end:
- if t.Indexes != nil {
- if len(t.Indexes) != len(r.a) {
- for i := 0; i < len(r.a); i++ {
- r.a[i].Index = 0
- }
- } else {
- for i := 0; i < len(r.a); i++ {
- r.a[i].Index = t.Indexes[i]
- }
- }
- }
- return
-}
-
-// Parse parses the json and returns a result.
-//
-// This function expects that the json is well-formed, and does not validate.
-// Invalid json will not panic, but it may return back unexpected results.
-// If you are consuming JSON from an unpredictable source then you may want to
-// use the Valid function first.
-func Parse(json string) Result {
- var value Result
- i := 0
- for ; i < len(json); i++ {
- if json[i] == '{' || json[i] == '[' {
- value.Type = JSON
- value.Raw = json[i:] // just take the entire raw
- break
- }
- if json[i] <= ' ' {
- continue
- }
- switch json[i] {
- case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
- 'i', 'I', 'N':
- value.Type = Number
- value.Raw, value.Num = tonum(json[i:])
- case 'n':
- if i+1 < len(json) && json[i+1] != 'u' {
- // nan
- value.Type = Number
- value.Raw, value.Num = tonum(json[i:])
- } else {
- // null
- value.Type = Null
- value.Raw = tolit(json[i:])
- }
- case 't':
- value.Type = True
- value.Raw = tolit(json[i:])
- case 'f':
- value.Type = False
- value.Raw = tolit(json[i:])
- case '"':
- value.Type = String
- value.Raw, value.Str = tostr(json[i:])
- default:
- return Result{}
- }
- break
- }
- if value.Exists() {
- value.Index = i
- }
- return value
-}
-
-// ParseBytes parses the json and returns a result.
-// If working with bytes, this method preferred over Parse(string(data))
-func ParseBytes(json []byte) Result {
- return Parse(string(json))
-}
-
-func squash(json string) string {
- // expects that the lead character is a '[' or '{' or '(' or '"'
- // squash the value, ignoring all nested arrays and objects.
- var i, depth int
- if json[0] != '"' {
- i, depth = 1, 1
- }
- for ; i < len(json); i++ {
- if json[i] >= '"' && json[i] <= '}' {
- switch json[i] {
- case '"':
- i++
- s2 := i
- for ; i < len(json); i++ {
- if json[i] > '\\' {
- continue
- }
- if json[i] == '"' {
- // look for an escaped slash
- if json[i-1] == '\\' {
- n := 0
- for j := i - 2; j > s2-1; j-- {
- if json[j] != '\\' {
- break
- }
- n++
- }
- if n%2 == 0 {
- continue
- }
- }
- break
- }
- }
- if depth == 0 {
- if i >= len(json) {
- return json
- }
- return json[:i+1]
- }
- case '{', '[', '(':
- depth++
- case '}', ']', ')':
- depth--
- if depth == 0 {
- return json[:i+1]
- }
- }
- }
- }
- return json
-}
-
-func tonum(json string) (raw string, num float64) {
- for i := 1; i < len(json); i++ {
- // less than dash might have valid characters
- if json[i] <= '-' {
- if json[i] <= ' ' || json[i] == ',' {
- // break on whitespace and comma
- raw = json[:i]
- num, _ = strconv.ParseFloat(raw, 64)
- return
- }
- // could be a '+' or '-'. let's assume so.
- } else if json[i] == ']' || json[i] == '}' {
- // break on ']' or '}'
- raw = json[:i]
- num, _ = strconv.ParseFloat(raw, 64)
- return
- }
- }
- raw = json
- num, _ = strconv.ParseFloat(raw, 64)
- return
-}
-
-func tolit(json string) (raw string) {
- for i := 1; i < len(json); i++ {
- if json[i] < 'a' || json[i] > 'z' {
- return json[:i]
- }
- }
- return json
-}
-
-func tostr(json string) (raw string, str string) {
- // expects that the lead character is a '"'
- for i := 1; i < len(json); i++ {
- if json[i] > '\\' {
- continue
- }
- if json[i] == '"' {
- return json[:i+1], json[1:i]
- }
- if json[i] == '\\' {
- i++
- for ; i < len(json); i++ {
- if json[i] > '\\' {
- continue
- }
- if json[i] == '"' {
- // look for an escaped slash
- if json[i-1] == '\\' {
- n := 0
- for j := i - 2; j > 0; j-- {
- if json[j] != '\\' {
- break
- }
- n++
- }
- if n%2 == 0 {
- continue
- }
- }
- return json[:i+1], unescape(json[1:i])
- }
- }
- var ret string
- if i+1 < len(json) {
- ret = json[:i+1]
- } else {
- ret = json[:i]
- }
- return ret, unescape(json[1:i])
- }
- }
- return json, json[1:]
-}
-
-// Exists returns true if value exists.
-//
-// if gjson.Get(json, "name.last").Exists(){
-// println("value exists")
-// }
-func (t Result) Exists() bool {
- return t.Type != Null || len(t.Raw) != 0
-}
-
-// Value returns one of these types:
-//
-// bool, for JSON booleans
-// float64, for JSON numbers
-// Number, for JSON numbers
-// string, for JSON string literals
-// nil, for JSON null
-// map[string]interface{}, for JSON objects
-// []interface{}, for JSON arrays
-func (t Result) Value() interface{} {
- if t.Type == String {
- return t.Str
- }
- switch t.Type {
- default:
- return nil
- case False:
- return false
- case Number:
- return t.Num
- case JSON:
- r := t.arrayOrMap(0, true)
- if r.vc == '{' {
- return r.oi
- } else if r.vc == '[' {
- return r.ai
- }
- return nil
- case True:
- return true
- }
-}
-
-func parseString(json string, i int) (int, string, bool, bool) {
- var s = i
- for ; i < len(json); i++ {
- if json[i] > '\\' {
- continue
- }
- if json[i] == '"' {
- return i + 1, json[s-1 : i+1], false, true
- }
- if json[i] == '\\' {
- i++
- for ; i < len(json); i++ {
- if json[i] > '\\' {
- continue
- }
- if json[i] == '"' {
- // look for an escaped slash
- if json[i-1] == '\\' {
- n := 0
- for j := i - 2; j > 0; j-- {
- if json[j] != '\\' {
- break
- }
- n++
- }
- if n%2 == 0 {
- continue
- }
- }
- return i + 1, json[s-1 : i+1], true, true
- }
- }
- break
- }
- }
- return i, json[s-1:], false, false
-}
-
-func parseNumber(json string, i int) (int, string) {
- var s = i
- i++
- for ; i < len(json); i++ {
- if json[i] <= ' ' || json[i] == ',' || json[i] == ']' ||
- json[i] == '}' {
- return i, json[s:i]
- }
- }
- return i, json[s:]
-}
-
-func parseLiteral(json string, i int) (int, string) {
- var s = i
- i++
- for ; i < len(json); i++ {
- if json[i] < 'a' || json[i] > 'z' {
- return i, json[s:i]
- }
- }
- return i, json[s:]
-}
-
-type arrayPathResult struct {
- part string
- path string
- pipe string
- piped bool
- more bool
- alogok bool
- arrch bool
- alogkey string
- query struct {
- on bool
- all bool
- path string
- op string
- value string
- }
-}
-
-func parseArrayPath(path string) (r arrayPathResult) {
- for i := 0; i < len(path); i++ {
- if path[i] == '|' {
- r.part = path[:i]
- r.pipe = path[i+1:]
- r.piped = true
- return
- }
- if path[i] == '.' {
- r.part = path[:i]
- if !r.arrch && i < len(path)-1 && isDotPiperChar(path[i+1:]) {
- r.pipe = path[i+1:]
- r.piped = true
- } else {
- r.path = path[i+1:]
- r.more = true
- }
- return
- }
- if path[i] == '#' {
- r.arrch = true
- if i == 0 && len(path) > 1 {
- if path[1] == '.' {
- r.alogok = true
- r.alogkey = path[2:]
- r.path = path[:1]
- } else if path[1] == '[' || path[1] == '(' {
- // query
- r.query.on = true
- qpath, op, value, _, fi, vesc, ok :=
- parseQuery(path[i:])
- if !ok {
- // bad query, end now
- break
- }
- if len(value) >= 2 && value[0] == '"' &&
- value[len(value)-1] == '"' {
- value = value[1 : len(value)-1]
- if vesc {
- value = unescape(value)
- }
- }
- r.query.path = qpath
- r.query.op = op
- r.query.value = value
-
- i = fi - 1
- if i+1 < len(path) && path[i+1] == '#' {
- r.query.all = true
- }
- }
- }
- continue
- }
- }
- r.part = path
- r.path = ""
- return
-}
-
-// splitQuery takes a query and splits it into three parts:
-//
-// path, op, middle, and right.
-//
-// So for this query:
-//
-// #(first_name=="Murphy").last
-//
-// Becomes
-//
-// first_name # path
-// =="Murphy" # middle
-// .last # right
-//
-// Or,
-//
-// #(service_roles.#(=="one")).cap
-//
-// Becomes
-//
-// service_roles.#(=="one") # path
-// # middle
-// .cap # right
-func parseQuery(query string) (
- path, op, value, remain string, i int, vesc, ok bool,
-) {
- if len(query) < 2 || query[0] != '#' ||
- (query[1] != '(' && query[1] != '[') {
- return "", "", "", "", i, false, false
- }
- i = 2
- j := 0 // start of value part
- depth := 1
- for ; i < len(query); i++ {
- if depth == 1 && j == 0 {
- switch query[i] {
- case '!', '=', '<', '>', '%':
- // start of the value part
- j = i
- continue
- }
- }
- if query[i] == '\\' {
- i++
- } else if query[i] == '[' || query[i] == '(' {
- depth++
- } else if query[i] == ']' || query[i] == ')' {
- depth--
- if depth == 0 {
- break
- }
- } else if query[i] == '"' {
- // inside selector string, balance quotes
- i++
- for ; i < len(query); i++ {
- if query[i] == '\\' {
- vesc = true
- i++
- } else if query[i] == '"' {
- break
- }
- }
- }
- }
- if depth > 0 {
- return "", "", "", "", i, false, false
- }
- if j > 0 {
- path = trim(query[2:j])
- value = trim(query[j:i])
- remain = query[i+1:]
- // parse the compare op from the value
- var opsz int
- switch {
- case len(value) == 1:
- opsz = 1
- case value[0] == '!' && value[1] == '=':
- opsz = 2
- case value[0] == '!' && value[1] == '%':
- opsz = 2
- case value[0] == '<' && value[1] == '=':
- opsz = 2
- case value[0] == '>' && value[1] == '=':
- opsz = 2
- case value[0] == '=' && value[1] == '=':
- value = value[1:]
- opsz = 1
- case value[0] == '<':
- opsz = 1
- case value[0] == '>':
- opsz = 1
- case value[0] == '=':
- opsz = 1
- case value[0] == '%':
- opsz = 1
- }
- op = value[:opsz]
- value = trim(value[opsz:])
- } else {
- path = trim(query[2:i])
- remain = query[i+1:]
- }
- return path, op, value, remain, i + 1, vesc, true
-}
-
-func trim(s string) string {
-left:
- if len(s) > 0 && s[0] <= ' ' {
- s = s[1:]
- goto left
- }
-right:
- if len(s) > 0 && s[len(s)-1] <= ' ' {
- s = s[:len(s)-1]
- goto right
- }
- return s
-}
-
-// peek at the next byte and see if it's a '@', '[', or '{'.
-func isDotPiperChar(s string) bool {
- if DisableModifiers {
- return false
- }
- c := s[0]
- if c == '@' {
- // check that the next component is *not* a modifier.
- i := 1
- for ; i < len(s); i++ {
- if s[i] == '.' || s[i] == '|' || s[i] == ':' {
- break
- }
- }
- _, ok := modifiers[s[1:i]]
- return ok
- }
- return c == '[' || c == '{'
-}
-
-type objectPathResult struct {
- part string
- path string
- pipe string
- piped bool
- wild bool
- more bool
-}
-
-func parseObjectPath(path string) (r objectPathResult) {
- for i := 0; i < len(path); i++ {
- if path[i] == '|' {
- r.part = path[:i]
- r.pipe = path[i+1:]
- r.piped = true
- return
- }
- if path[i] == '.' {
- r.part = path[:i]
- if i < len(path)-1 && isDotPiperChar(path[i+1:]) {
- r.pipe = path[i+1:]
- r.piped = true
- } else {
- r.path = path[i+1:]
- r.more = true
- }
- return
- }
- if path[i] == '*' || path[i] == '?' {
- r.wild = true
- continue
- }
- if path[i] == '\\' {
- // go into escape mode. this is a slower path that
- // strips off the escape character from the part.
- epart := []byte(path[:i])
- i++
- if i < len(path) {
- epart = append(epart, path[i])
- i++
- for ; i < len(path); i++ {
- if path[i] == '\\' {
- i++
- if i < len(path) {
- epart = append(epart, path[i])
- }
- continue
- } else if path[i] == '.' {
- r.part = string(epart)
- if i < len(path)-1 && isDotPiperChar(path[i+1:]) {
- r.pipe = path[i+1:]
- r.piped = true
- } else {
- r.path = path[i+1:]
- r.more = true
- }
- return
- } else if path[i] == '|' {
- r.part = string(epart)
- r.pipe = path[i+1:]
- r.piped = true
- return
- } else if path[i] == '*' || path[i] == '?' {
- r.wild = true
- }
- epart = append(epart, path[i])
- }
- }
- // append the last part
- r.part = string(epart)
- return
- }
- }
- r.part = path
- return
-}
-
-var vchars = [256]byte{
- '"': 2, '{': 3, '(': 3, '[': 3, '}': 1, ')': 1, ']': 1,
-}
-
-func parseSquash(json string, i int) (int, string) {
- // expects that the lead character is a '[' or '{' or '('
- // squash the value, ignoring all nested arrays and objects.
- // the first '[' or '{' or '(' has already been read
- s := i
- i++
- depth := 1
- var c byte
- for i < len(json) {
- for i < len(json)-8 {
- jslice := json[i : i+8]
- c = vchars[jslice[0]]
- if c != 0 {
- i += 0
- goto token
- }
- c = vchars[jslice[1]]
- if c != 0 {
- i += 1
- goto token
- }
- c = vchars[jslice[2]]
- if c != 0 {
- i += 2
- goto token
- }
- c = vchars[jslice[3]]
- if c != 0 {
- i += 3
- goto token
- }
- c = vchars[jslice[4]]
- if c != 0 {
- i += 4
- goto token
- }
- c = vchars[jslice[5]]
- if c != 0 {
- i += 5
- goto token
- }
- c = vchars[jslice[6]]
- if c != 0 {
- i += 6
- goto token
- }
- c = vchars[jslice[7]]
- if c != 0 {
- i += 7
- goto token
- }
- i += 8
- }
- c = vchars[json[i]]
- if c == 0 {
- i++
- continue
- }
- token:
- if c == 2 {
- // '"' string
- i++
- s2 := i
- nextquote:
- for i < len(json)-8 {
- jslice := json[i : i+8]
- if jslice[0] == '"' {
- i += 0
- goto strchkesc
- }
- if jslice[1] == '"' {
- i += 1
- goto strchkesc
- }
- if jslice[2] == '"' {
- i += 2
- goto strchkesc
- }
- if jslice[3] == '"' {
- i += 3
- goto strchkesc
- }
- if jslice[4] == '"' {
- i += 4
- goto strchkesc
- }
- if jslice[5] == '"' {
- i += 5
- goto strchkesc
- }
- if jslice[6] == '"' {
- i += 6
- goto strchkesc
- }
- if jslice[7] == '"' {
- i += 7
- goto strchkesc
- }
- i += 8
- }
- goto strchkstd
- strchkesc:
- if json[i-1] != '\\' {
- i++
- continue
- }
- strchkstd:
- for i < len(json) {
- if json[i] > '\\' || json[i] != '"' {
- i++
- continue
- }
- // look for an escaped slash
- if json[i-1] == '\\' {
- n := 0
- for j := i - 2; j > s2-1; j-- {
- if json[j] != '\\' {
- break
- }
- n++
- }
- if n%2 == 0 {
- i++
- goto nextquote
- }
- }
- break
- }
- } else {
- // '{', '[', '(', '}', ']', ')'
- // open close tokens
- depth += int(c) - 2
- if depth == 0 {
- i++
- return i, json[s:i]
- }
- }
- i++
- }
- return i, json[s:]
-}
-
-func parseObject(c *parseContext, i int, path string) (int, bool) {
- var pmatch, kesc, vesc, ok, hit bool
- var key, val string
- rp := parseObjectPath(path)
- if !rp.more && rp.piped {
- c.pipe = rp.pipe
- c.piped = true
- }
- for i < len(c.json) {
- for ; i < len(c.json); i++ {
- if c.json[i] == '"' {
- // parse_key_string
- // this is slightly different from getting s string value
- // because we don't need the outer quotes.
- i++
- var s = i
- for ; i < len(c.json); i++ {
- if c.json[i] > '\\' {
- continue
- }
- if c.json[i] == '"' {
- i, key, kesc, ok = i+1, c.json[s:i], false, true
- goto parse_key_string_done
- }
- if c.json[i] == '\\' {
- i++
- for ; i < len(c.json); i++ {
- if c.json[i] > '\\' {
- continue
- }
- if c.json[i] == '"' {
- // look for an escaped slash
- if c.json[i-1] == '\\' {
- n := 0
- for j := i - 2; j > 0; j-- {
- if c.json[j] != '\\' {
- break
- }
- n++
- }
- if n%2 == 0 {
- continue
- }
- }
- i, key, kesc, ok = i+1, c.json[s:i], true, true
- goto parse_key_string_done
- }
- }
- break
- }
- }
- key, kesc, ok = c.json[s:], false, false
- parse_key_string_done:
- break
- }
- if c.json[i] == '}' {
- return i + 1, false
- }
- }
- if !ok {
- return i, false
- }
- if rp.wild {
- if kesc {
- pmatch = matchLimit(unescape(key), rp.part)
- } else {
- pmatch = matchLimit(key, rp.part)
- }
- } else {
- if kesc {
- pmatch = rp.part == unescape(key)
- } else {
- pmatch = rp.part == key
- }
- }
- hit = pmatch && !rp.more
- for ; i < len(c.json); i++ {
- var num bool
- switch c.json[i] {
- default:
- continue
- case '"':
- i++
- i, val, vesc, ok = parseString(c.json, i)
- if !ok {
- return i, false
- }
- if hit {
- if vesc {
- c.value.Str = unescape(val[1 : len(val)-1])
- } else {
- c.value.Str = val[1 : len(val)-1]
- }
- c.value.Raw = val
- c.value.Type = String
- return i, true
- }
- case '{':
- if pmatch && !hit {
- i, hit = parseObject(c, i+1, rp.path)
- if hit {
- return i, true
- }
- } else {
- i, val = parseSquash(c.json, i)
- if hit {
- c.value.Raw = val
- c.value.Type = JSON
- return i, true
- }
- }
- case '[':
- if pmatch && !hit {
- i, hit = parseArray(c, i+1, rp.path)
- if hit {
- return i, true
- }
- } else {
- i, val = parseSquash(c.json, i)
- if hit {
- c.value.Raw = val
- c.value.Type = JSON
- return i, true
- }
- }
- case 'n':
- if i+1 < len(c.json) && c.json[i+1] != 'u' {
- num = true
- break
- }
- fallthrough
- case 't', 'f':
- vc := c.json[i]
- i, val = parseLiteral(c.json, i)
- if hit {
- c.value.Raw = val
- switch vc {
- case 't':
- c.value.Type = True
- case 'f':
- c.value.Type = False
- }
- return i, true
- }
- case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
- 'i', 'I', 'N':
- num = true
- }
- if num {
- i, val = parseNumber(c.json, i)
- if hit {
- c.value.Raw = val
- c.value.Type = Number
- c.value.Num, _ = strconv.ParseFloat(val, 64)
- return i, true
- }
- }
- break
- }
- }
- return i, false
-}
-
-// matchLimit will limit the complexity of the match operation to avoid ReDos
-// attacks from arbitrary inputs.
-// See the github.com/tidwall/match.MatchLimit function for more information.
-func matchLimit(str, pattern string) bool {
- matched, _ := match.MatchLimit(str, pattern, 10000)
- return matched
-}
-
-func falseish(t Result) bool {
- switch t.Type {
- case Null:
- return true
- case False:
- return true
- case String:
- b, err := strconv.ParseBool(strings.ToLower(t.Str))
- if err != nil {
- return false
- }
- return !b
- case Number:
- return t.Num == 0
- default:
- return false
- }
-}
-
-func trueish(t Result) bool {
- switch t.Type {
- case True:
- return true
- case String:
- b, err := strconv.ParseBool(strings.ToLower(t.Str))
- if err != nil {
- return false
- }
- return b
- case Number:
- return t.Num != 0
- default:
- return false
- }
-}
-
-func nullish(t Result) bool {
- return t.Type == Null
-}
-
-func queryMatches(rp *arrayPathResult, value Result) bool {
- rpv := rp.query.value
- if len(rpv) > 0 {
- if rpv[0] == '~' {
- // convert to bool
- rpv = rpv[1:]
- var ish, ok bool
- switch rpv {
- case "*":
- ish, ok = value.Exists(), true
- case "null":
- ish, ok = nullish(value), true
- case "true":
- ish, ok = trueish(value), true
- case "false":
- ish, ok = falseish(value), true
- }
- if ok {
- rpv = "true"
- if ish {
- value = Result{Type: True}
- } else {
- value = Result{Type: False}
- }
- } else {
- rpv = ""
- value = Result{}
- }
- }
- }
- if !value.Exists() {
- return false
- }
- if rp.query.op == "" {
- // the query is only looking for existence, such as:
- // friends.#(name)
- // which makes sure that the array "friends" has an element of
- // "name" that exists
- return true
- }
- switch value.Type {
- case String:
- switch rp.query.op {
- case "=":
- return value.Str == rpv
- case "!=":
- return value.Str != rpv
- case "<":
- return value.Str < rpv
- case "<=":
- return value.Str <= rpv
- case ">":
- return value.Str > rpv
- case ">=":
- return value.Str >= rpv
- case "%":
- return matchLimit(value.Str, rpv)
- case "!%":
- return !matchLimit(value.Str, rpv)
- }
- case Number:
- rpvn, _ := strconv.ParseFloat(rpv, 64)
- switch rp.query.op {
- case "=":
- return value.Num == rpvn
- case "!=":
- return value.Num != rpvn
- case "<":
- return value.Num < rpvn
- case "<=":
- return value.Num <= rpvn
- case ">":
- return value.Num > rpvn
- case ">=":
- return value.Num >= rpvn
- }
- case True:
- switch rp.query.op {
- case "=":
- return rpv == "true"
- case "!=":
- return rpv != "true"
- case ">":
- return rpv == "false"
- case ">=":
- return true
- }
- case False:
- switch rp.query.op {
- case "=":
- return rpv == "false"
- case "!=":
- return rpv != "false"
- case "<":
- return rpv == "true"
- case "<=":
- return true
- }
- }
- return false
-}
-func parseArray(c *parseContext, i int, path string) (int, bool) {
- var pmatch, vesc, ok, hit bool
- var val string
- var h int
- var alog []int
- var partidx int
- var multires []byte
- var queryIndexes []int
- rp := parseArrayPath(path)
- if !rp.arrch {
- n, ok := parseUint(rp.part)
- if !ok {
- partidx = -1
- } else {
- partidx = int(n)
- }
- }
- if !rp.more && rp.piped {
- c.pipe = rp.pipe
- c.piped = true
- }
-
- procQuery := func(qval Result) bool {
- if rp.query.all {
- if len(multires) == 0 {
- multires = append(multires, '[')
- }
- }
- var tmp parseContext
- tmp.value = qval
- fillIndex(c.json, &tmp)
- parentIndex := tmp.value.Index
- var res Result
- if qval.Type == JSON {
- res = qval.Get(rp.query.path)
- } else {
- if rp.query.path != "" {
- return false
- }
- res = qval
- }
- if queryMatches(&rp, res) {
- if rp.more {
- left, right, ok := splitPossiblePipe(rp.path)
- if ok {
- rp.path = left
- c.pipe = right
- c.piped = true
- }
- res = qval.Get(rp.path)
- } else {
- res = qval
- }
- if rp.query.all {
- raw := res.Raw
- if len(raw) == 0 {
- raw = res.String()
- }
- if raw != "" {
- if len(multires) > 1 {
- multires = append(multires, ',')
- }
- multires = append(multires, raw...)
- queryIndexes = append(queryIndexes, res.Index+parentIndex)
- }
- } else {
- c.value = res
- return true
- }
- }
- return false
- }
- for i < len(c.json)+1 {
- if !rp.arrch {
- pmatch = partidx == h
- hit = pmatch && !rp.more
- }
- h++
- if rp.alogok {
- alog = append(alog, i)
- }
- for ; ; i++ {
- var ch byte
- if i > len(c.json) {
- break
- } else if i == len(c.json) {
- ch = ']'
- } else {
- ch = c.json[i]
- }
- var num bool
- switch ch {
- default:
- continue
- case '"':
- i++
- i, val, vesc, ok = parseString(c.json, i)
- if !ok {
- return i, false
- }
- if rp.query.on {
- var qval Result
- if vesc {
- qval.Str = unescape(val[1 : len(val)-1])
- } else {
- qval.Str = val[1 : len(val)-1]
- }
- qval.Raw = val
- qval.Type = String
- if procQuery(qval) {
- return i, true
- }
- } else if hit {
- if rp.alogok {
- break
- }
- if vesc {
- c.value.Str = unescape(val[1 : len(val)-1])
- } else {
- c.value.Str = val[1 : len(val)-1]
- }
- c.value.Raw = val
- c.value.Type = String
- return i, true
- }
- case '{':
- if pmatch && !hit {
- i, hit = parseObject(c, i+1, rp.path)
- if hit {
- if rp.alogok {
- break
- }
- return i, true
- }
- } else {
- i, val = parseSquash(c.json, i)
- if rp.query.on {
- if procQuery(Result{Raw: val, Type: JSON}) {
- return i, true
- }
- } else if hit {
- if rp.alogok {
- break
- }
- c.value.Raw = val
- c.value.Type = JSON
- return i, true
- }
- }
- case '[':
- if pmatch && !hit {
- i, hit = parseArray(c, i+1, rp.path)
- if hit {
- if rp.alogok {
- break
- }
- return i, true
- }
- } else {
- i, val = parseSquash(c.json, i)
- if rp.query.on {
- if procQuery(Result{Raw: val, Type: JSON}) {
- return i, true
- }
- } else if hit {
- if rp.alogok {
- break
- }
- c.value.Raw = val
- c.value.Type = JSON
- return i, true
- }
- }
- case 'n':
- if i+1 < len(c.json) && c.json[i+1] != 'u' {
- num = true
- break
- }
- fallthrough
- case 't', 'f':
- vc := c.json[i]
- i, val = parseLiteral(c.json, i)
- if rp.query.on {
- var qval Result
- qval.Raw = val
- switch vc {
- case 't':
- qval.Type = True
- case 'f':
- qval.Type = False
- }
- if procQuery(qval) {
- return i, true
- }
- } else if hit {
- if rp.alogok {
- break
- }
- c.value.Raw = val
- switch vc {
- case 't':
- c.value.Type = True
- case 'f':
- c.value.Type = False
- }
- return i, true
- }
- case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
- 'i', 'I', 'N':
- num = true
- case ']':
- if rp.arrch && rp.part == "#" {
- if rp.alogok {
- left, right, ok := splitPossiblePipe(rp.alogkey)
- if ok {
- rp.alogkey = left
- c.pipe = right
- c.piped = true
- }
- var indexes = make([]int, 0, 64)
- var jsons = make([]byte, 0, 64)
- jsons = append(jsons, '[')
- for j, k := 0, 0; j < len(alog); j++ {
- idx := alog[j]
- for idx < len(c.json) {
- switch c.json[idx] {
- case ' ', '\t', '\r', '\n':
- idx++
- continue
- }
- break
- }
- if idx < len(c.json) && c.json[idx] != ']' {
- _, res, ok := parseAny(c.json, idx, true)
- if ok {
- res := res.Get(rp.alogkey)
- if res.Exists() {
- if k > 0 {
- jsons = append(jsons, ',')
- }
- raw := res.Raw
- if len(raw) == 0 {
- raw = res.String()
- }
- jsons = append(jsons, []byte(raw)...)
- indexes = append(indexes, res.Index)
- k++
- }
- }
- }
- }
- jsons = append(jsons, ']')
- c.value.Type = JSON
- c.value.Raw = string(jsons)
- c.value.Indexes = indexes
- return i + 1, true
- }
- if rp.alogok {
- break
- }
-
- c.value.Type = Number
- c.value.Num = float64(h - 1)
- c.value.Raw = strconv.Itoa(h - 1)
- c.calcd = true
- return i + 1, true
- }
- if !c.value.Exists() {
- if len(multires) > 0 {
- c.value = Result{
- Raw: string(append(multires, ']')),
- Type: JSON,
- Indexes: queryIndexes,
- }
- } else if rp.query.all {
- c.value = Result{
- Raw: "[]",
- Type: JSON,
- }
- }
- }
- return i + 1, false
- }
- if num {
- i, val = parseNumber(c.json, i)
- if rp.query.on {
- var qval Result
- qval.Raw = val
- qval.Type = Number
- qval.Num, _ = strconv.ParseFloat(val, 64)
- if procQuery(qval) {
- return i, true
- }
- } else if hit {
- if rp.alogok {
- break
- }
- c.value.Raw = val
- c.value.Type = Number
- c.value.Num, _ = strconv.ParseFloat(val, 64)
- return i, true
- }
- }
- break
- }
- }
- return i, false
-}
-
-func splitPossiblePipe(path string) (left, right string, ok bool) {
- // take a quick peek for the pipe character. If found we'll split the piped
- // part of the path into the c.pipe field and shorten the rp.
- var possible bool
- for i := 0; i < len(path); i++ {
- if path[i] == '|' {
- possible = true
- break
- }
- }
- if !possible {
- return
- }
-
- if len(path) > 0 && path[0] == '{' {
- squashed := squash(path[1:])
- if len(squashed) < len(path)-1 {
- squashed = path[:len(squashed)+1]
- remain := path[len(squashed):]
- if remain[0] == '|' {
- return squashed, remain[1:], true
- }
- }
- return
- }
-
- // split the left and right side of the path with the pipe character as
- // the delimiter. This is a little tricky because we'll need to basically
- // parse the entire path.
- for i := 0; i < len(path); i++ {
- if path[i] == '\\' {
- i++
- } else if path[i] == '.' {
- if i == len(path)-1 {
- return
- }
- if path[i+1] == '#' {
- i += 2
- if i == len(path) {
- return
- }
- if path[i] == '[' || path[i] == '(' {
- var start, end byte
- if path[i] == '[' {
- start, end = '[', ']'
- } else {
- start, end = '(', ')'
- }
- // inside selector, balance brackets
- i++
- depth := 1
- for ; i < len(path); i++ {
- if path[i] == '\\' {
- i++
- } else if path[i] == start {
- depth++
- } else if path[i] == end {
- depth--
- if depth == 0 {
- break
- }
- } else if path[i] == '"' {
- // inside selector string, balance quotes
- i++
- for ; i < len(path); i++ {
- if path[i] == '\\' {
- i++
- } else if path[i] == '"' {
- break
- }
- }
- }
- }
- }
- }
- } else if path[i] == '|' {
- return path[:i], path[i+1:], true
- }
- }
- return
-}
-
-// ForEachLine iterates through lines of JSON as specified by the JSON Lines
-// format (http://jsonlines.org/).
-// Each line is returned as a GJSON Result.
-func ForEachLine(json string, iterator func(line Result) bool) {
- var res Result
- var i int
- for {
- i, res, _ = parseAny(json, i, true)
- if !res.Exists() {
- break
- }
- if !iterator(res) {
- return
- }
- }
-}
-
-type subSelector struct {
- name string
- path string
-}
-
-// parseSubSelectors returns the subselectors belonging to a '[path1,path2]' or
-// '{"field1":path1,"field2":path2}' type subSelection. It's expected that the
-// first character in path is either '[' or '{', and has already been checked
-// prior to calling this function.
-func parseSubSelectors(path string) (sels []subSelector, out string, ok bool) {
- modifier := 0
- depth := 1
- colon := 0
- start := 1
- i := 1
- pushSel := func() {
- var sel subSelector
- if colon == 0 {
- sel.path = path[start:i]
- } else {
- sel.name = path[start:colon]
- sel.path = path[colon+1 : i]
- }
- sels = append(sels, sel)
- colon = 0
- modifier = 0
- start = i + 1
- }
- for ; i < len(path); i++ {
- switch path[i] {
- case '\\':
- i++
- case '@':
- if modifier == 0 && i > 0 && (path[i-1] == '.' || path[i-1] == '|') {
- modifier = i
- }
- case ':':
- if modifier == 0 && colon == 0 && depth == 1 {
- colon = i
- }
- case ',':
- if depth == 1 {
- pushSel()
- }
- case '"':
- i++
- loop:
- for ; i < len(path); i++ {
- switch path[i] {
- case '\\':
- i++
- case '"':
- break loop
- }
- }
- case '[', '(', '{':
- depth++
- case ']', ')', '}':
- depth--
- if depth == 0 {
- pushSel()
- path = path[i+1:]
- return sels, path, true
- }
- }
- }
- return
-}
-
-// nameOfLast returns the name of the last component
-func nameOfLast(path string) string {
- for i := len(path) - 1; i >= 0; i-- {
- if path[i] == '|' || path[i] == '.' {
- if i > 0 {
- if path[i-1] == '\\' {
- continue
- }
- }
- return path[i+1:]
- }
- }
- return path
-}
-
-func isSimpleName(component string) bool {
- for i := 0; i < len(component); i++ {
- if component[i] < ' ' {
- return false
- }
- switch component[i] {
- case '[', ']', '{', '}', '(', ')', '#', '|', '!':
- return false
- }
- }
- return true
-}
-
-var hexchars = [...]byte{
- '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
- 'a', 'b', 'c', 'd', 'e', 'f',
-}
-
-func appendHex16(dst []byte, x uint16) []byte {
- return append(dst,
- hexchars[x>>12&0xF], hexchars[x>>8&0xF],
- hexchars[x>>4&0xF], hexchars[x>>0&0xF],
- )
-}
-
-// DisableEscapeHTML will disable the automatic escaping of certain
-// "problamatic" HTML characters when encoding to JSON.
-// These character include '>', '<' and '&', which get escaped to \u003e,
-// \u0026, and \u003c respectively.
-//
-// This is a global flag and will affect all further gjson operations.
-// Ideally, if used, it should be set one time before other gjson functions
-// are called.
-var DisableEscapeHTML = false
-
-// AppendJSONString is a convenience function that converts the provided string
-// to a valid JSON string and appends it to dst.
-func AppendJSONString(dst []byte, s string) []byte {
- dst = append(dst, make([]byte, len(s)+2)...)
- dst = append(dst[:len(dst)-len(s)-2], '"')
- for i := 0; i < len(s); i++ {
- if s[i] < ' ' {
- dst = append(dst, '\\')
- switch s[i] {
- case '\b':
- dst = append(dst, 'b')
- case '\f':
- dst = append(dst, 'f')
- case '\n':
- dst = append(dst, 'n')
- case '\r':
- dst = append(dst, 'r')
- case '\t':
- dst = append(dst, 't')
- default:
- dst = append(dst, 'u')
- dst = appendHex16(dst, uint16(s[i]))
- }
- } else if !DisableEscapeHTML &&
- (s[i] == '>' || s[i] == '<' || s[i] == '&') {
- dst = append(dst, '\\', 'u')
- dst = appendHex16(dst, uint16(s[i]))
- } else if s[i] == '\\' {
- dst = append(dst, '\\', '\\')
- } else if s[i] == '"' {
- dst = append(dst, '\\', '"')
- } else if s[i] > 127 {
- // read utf8 character
- r, n := utf8.DecodeRuneInString(s[i:])
- if n == 0 {
- break
- }
- if r == utf8.RuneError && n == 1 {
- dst = append(dst, `\ufffd`...)
- } else if r == '\u2028' || r == '\u2029' {
- dst = append(dst, `\u202`...)
- dst = append(dst, hexchars[r&0xF])
- } else {
- dst = append(dst, s[i:i+n]...)
- }
- i = i + n - 1
- } else {
- dst = append(dst, s[i])
- }
- }
- return append(dst, '"')
-}
-
-type parseContext struct {
- json string
- value Result
- pipe string
- piped bool
- calcd bool
- lines bool
-}
-
-// Get searches json for the specified path.
-// A path is in dot syntax, such as "name.last" or "age".
-// When the value is found it's returned immediately.
-//
-// A path is a series of keys separated by a dot.
-// A key may contain special wildcard characters '*' and '?'.
-// To access an array value use the index as the key.
-// To get the number of elements in an array or to access a child path, use
-// the '#' character.
-// The dot and wildcard character can be escaped with '\'.
-//
-// {
-// "name": {"first": "Tom", "last": "Anderson"},
-// "age":37,
-// "children": ["Sara","Alex","Jack"],
-// "friends": [
-// {"first": "James", "last": "Murphy"},
-// {"first": "Roger", "last": "Craig"}
-// ]
-// }
-// "name.last" >> "Anderson"
-// "age" >> 37
-// "children" >> ["Sara","Alex","Jack"]
-// "children.#" >> 3
-// "children.1" >> "Alex"
-// "child*.2" >> "Jack"
-// "c?ildren.0" >> "Sara"
-// "friends.#.first" >> ["James","Roger"]
-//
-// This function expects that the json is well-formed, and does not validate.
-// Invalid json will not panic, but it may return back unexpected results.
-// If you are consuming JSON from an unpredictable source then you may want to
-// use the Valid function first.
-func Get(json, path string) Result {
- if len(path) > 1 {
- if (path[0] == '@' && !DisableModifiers) || path[0] == '!' {
- // possible modifier
- var ok bool
- var npath string
- var rjson string
- if path[0] == '@' && !DisableModifiers {
- npath, rjson, ok = execModifier(json, path)
- } else if path[0] == '!' {
- npath, rjson, ok = execStatic(json, path)
- }
- if ok {
- path = npath
- if len(path) > 0 && (path[0] == '|' || path[0] == '.') {
- res := Get(rjson, path[1:])
- res.Index = 0
- res.Indexes = nil
- return res
- }
- return Parse(rjson)
- }
- }
- if path[0] == '[' || path[0] == '{' {
- // using a subselector path
- kind := path[0]
- var ok bool
- var subs []subSelector
- subs, path, ok = parseSubSelectors(path)
- if ok {
- if len(path) == 0 || (path[0] == '|' || path[0] == '.') {
- var b []byte
- b = append(b, kind)
- var i int
- for _, sub := range subs {
- res := Get(json, sub.path)
- if res.Exists() {
- if i > 0 {
- b = append(b, ',')
- }
- if kind == '{' {
- if len(sub.name) > 0 {
- if sub.name[0] == '"' && Valid(sub.name) {
- b = append(b, sub.name...)
- } else {
- b = AppendJSONString(b, sub.name)
- }
- } else {
- last := nameOfLast(sub.path)
- if isSimpleName(last) {
- b = AppendJSONString(b, last)
- } else {
- b = AppendJSONString(b, "_")
- }
- }
- b = append(b, ':')
- }
- var raw string
- if len(res.Raw) == 0 {
- raw = res.String()
- if len(raw) == 0 {
- raw = "null"
- }
- } else {
- raw = res.Raw
- }
- b = append(b, raw...)
- i++
- }
- }
- b = append(b, kind+2)
- var res Result
- res.Raw = string(b)
- res.Type = JSON
- if len(path) > 0 {
- res = res.Get(path[1:])
- }
- res.Index = 0
- return res
- }
- }
- }
- }
- var i int
- var c = &parseContext{json: json}
- if len(path) >= 2 && path[0] == '.' && path[1] == '.' {
- c.lines = true
- parseArray(c, 0, path[2:])
- } else {
- for ; i < len(c.json); i++ {
- if c.json[i] == '{' {
- i++
- parseObject(c, i, path)
- break
- }
- if c.json[i] == '[' {
- i++
- parseArray(c, i, path)
- break
- }
- }
- }
- if c.piped {
- res := c.value.Get(c.pipe)
- res.Index = 0
- return res
- }
- fillIndex(json, c)
- return c.value
-}
-
-// GetBytes searches json for the specified path.
-// If working with bytes, this method preferred over Get(string(data), path)
-func GetBytes(json []byte, path string) Result {
- return getBytes(json, path)
-}
-
-// runeit returns the rune from the the \uXXXX
-func runeit(json string) rune {
- n, _ := strconv.ParseUint(json[:4], 16, 64)
- return rune(n)
-}
-
-// unescape unescapes a string
-func unescape(json string) string {
- var str = make([]byte, 0, len(json))
- for i := 0; i < len(json); i++ {
- switch {
- default:
- str = append(str, json[i])
- case json[i] < ' ':
- return string(str)
- case json[i] == '\\':
- i++
- if i >= len(json) {
- return string(str)
- }
- switch json[i] {
- default:
- return string(str)
- case '\\':
- str = append(str, '\\')
- case '/':
- str = append(str, '/')
- case 'b':
- str = append(str, '\b')
- case 'f':
- str = append(str, '\f')
- case 'n':
- str = append(str, '\n')
- case 'r':
- str = append(str, '\r')
- case 't':
- str = append(str, '\t')
- case '"':
- str = append(str, '"')
- case 'u':
- if i+5 > len(json) {
- return string(str)
- }
- r := runeit(json[i+1:])
- i += 5
- if utf16.IsSurrogate(r) {
- // need another code
- if len(json[i:]) >= 6 && json[i] == '\\' &&
- json[i+1] == 'u' {
- // we expect it to be correct so just consume it
- r = utf16.DecodeRune(r, runeit(json[i+2:]))
- i += 6
- }
- }
- // provide enough space to encode the largest utf8 possible
- str = append(str, 0, 0, 0, 0, 0, 0, 0, 0)
- n := utf8.EncodeRune(str[len(str)-8:], r)
- str = str[:len(str)-8+n]
- i-- // backtrack index by one
- }
- }
- }
- return string(str)
-}
-
-// Less return true if a token is less than another token.
-// The caseSensitive parameter is used when the tokens are Strings.
-// The order when comparing two different type is:
-//
-// Null < False < Number < String < True < JSON
-func (t Result) Less(token Result, caseSensitive bool) bool {
- if t.Type < token.Type {
- return true
- }
- if t.Type > token.Type {
- return false
- }
- if t.Type == String {
- if caseSensitive {
- return t.Str < token.Str
- }
- return stringLessInsensitive(t.Str, token.Str)
- }
- if t.Type == Number {
- return t.Num < token.Num
- }
- return t.Raw < token.Raw
-}
-
-func stringLessInsensitive(a, b string) bool {
- for i := 0; i < len(a) && i < len(b); i++ {
- if a[i] >= 'A' && a[i] <= 'Z' {
- if b[i] >= 'A' && b[i] <= 'Z' {
- // both are uppercase, do nothing
- if a[i] < b[i] {
- return true
- } else if a[i] > b[i] {
- return false
- }
- } else {
- // a is uppercase, convert a to lowercase
- if a[i]+32 < b[i] {
- return true
- } else if a[i]+32 > b[i] {
- return false
- }
- }
- } else if b[i] >= 'A' && b[i] <= 'Z' {
- // b is uppercase, convert b to lowercase
- if a[i] < b[i]+32 {
- return true
- } else if a[i] > b[i]+32 {
- return false
- }
- } else {
- // neither are uppercase
- if a[i] < b[i] {
- return true
- } else if a[i] > b[i] {
- return false
- }
- }
- }
- return len(a) < len(b)
-}
-
-// parseAny parses the next value from a json string.
-// A Result is returned when the hit param is set.
-// The return values are (i int, res Result, ok bool)
-func parseAny(json string, i int, hit bool) (int, Result, bool) {
- var res Result
- var val string
- for ; i < len(json); i++ {
- if json[i] == '{' || json[i] == '[' {
- i, val = parseSquash(json, i)
- if hit {
- res.Raw = val
- res.Type = JSON
- }
- var tmp parseContext
- tmp.value = res
- fillIndex(json, &tmp)
- return i, tmp.value, true
- }
- if json[i] <= ' ' {
- continue
- }
- var num bool
- switch json[i] {
- case '"':
- i++
- var vesc bool
- var ok bool
- i, val, vesc, ok = parseString(json, i)
- if !ok {
- return i, res, false
- }
- if hit {
- res.Type = String
- res.Raw = val
- if vesc {
- res.Str = unescape(val[1 : len(val)-1])
- } else {
- res.Str = val[1 : len(val)-1]
- }
- }
- return i, res, true
- case 'n':
- if i+1 < len(json) && json[i+1] != 'u' {
- num = true
- break
- }
- fallthrough
- case 't', 'f':
- vc := json[i]
- i, val = parseLiteral(json, i)
- if hit {
- res.Raw = val
- switch vc {
- case 't':
- res.Type = True
- case 'f':
- res.Type = False
- }
- return i, res, true
- }
- case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
- 'i', 'I', 'N':
- num = true
- }
- if num {
- i, val = parseNumber(json, i)
- if hit {
- res.Raw = val
- res.Type = Number
- res.Num, _ = strconv.ParseFloat(val, 64)
- }
- return i, res, true
- }
-
- }
- return i, res, false
-}
-
-// GetMany searches json for the multiple paths.
-// The return value is a Result array where the number of items
-// will be equal to the number of input paths.
-func GetMany(json string, path ...string) []Result {
- res := make([]Result, len(path))
- for i, path := range path {
- res[i] = Get(json, path)
- }
- return res
-}
-
-// GetManyBytes searches json for the multiple paths.
-// The return value is a Result array where the number of items
-// will be equal to the number of input paths.
-func GetManyBytes(json []byte, path ...string) []Result {
- res := make([]Result, len(path))
- for i, path := range path {
- res[i] = GetBytes(json, path)
- }
- return res
-}
-
-func validpayload(data []byte, i int) (outi int, ok bool) {
- for ; i < len(data); i++ {
- switch data[i] {
- default:
- i, ok = validany(data, i)
- if !ok {
- return i, false
- }
- for ; i < len(data); i++ {
- switch data[i] {
- default:
- return i, false
- case ' ', '\t', '\n', '\r':
- continue
- }
- }
- return i, true
- case ' ', '\t', '\n', '\r':
- continue
- }
- }
- return i, false
-}
-func validany(data []byte, i int) (outi int, ok bool) {
- for ; i < len(data); i++ {
- switch data[i] {
- default:
- return i, false
- case ' ', '\t', '\n', '\r':
- continue
- case '{':
- return validobject(data, i+1)
- case '[':
- return validarray(data, i+1)
- case '"':
- return validstring(data, i+1)
- case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
- return validnumber(data, i+1)
- case 't':
- return validtrue(data, i+1)
- case 'f':
- return validfalse(data, i+1)
- case 'n':
- return validnull(data, i+1)
- }
- }
- return i, false
-}
-func validobject(data []byte, i int) (outi int, ok bool) {
- for ; i < len(data); i++ {
- switch data[i] {
- default:
- return i, false
- case ' ', '\t', '\n', '\r':
- continue
- case '}':
- return i + 1, true
- case '"':
- key:
- if i, ok = validstring(data, i+1); !ok {
- return i, false
- }
- if i, ok = validcolon(data, i); !ok {
- return i, false
- }
- if i, ok = validany(data, i); !ok {
- return i, false
- }
- if i, ok = validcomma(data, i, '}'); !ok {
- return i, false
- }
- if data[i] == '}' {
- return i + 1, true
- }
- i++
- for ; i < len(data); i++ {
- switch data[i] {
- default:
- return i, false
- case ' ', '\t', '\n', '\r':
- continue
- case '"':
- goto key
- }
- }
- return i, false
- }
- }
- return i, false
-}
-func validcolon(data []byte, i int) (outi int, ok bool) {
- for ; i < len(data); i++ {
- switch data[i] {
- default:
- return i, false
- case ' ', '\t', '\n', '\r':
- continue
- case ':':
- return i + 1, true
- }
- }
- return i, false
-}
-func validcomma(data []byte, i int, end byte) (outi int, ok bool) {
- for ; i < len(data); i++ {
- switch data[i] {
- default:
- return i, false
- case ' ', '\t', '\n', '\r':
- continue
- case ',':
- return i, true
- case end:
- return i, true
- }
- }
- return i, false
-}
-func validarray(data []byte, i int) (outi int, ok bool) {
- for ; i < len(data); i++ {
- switch data[i] {
- default:
- for ; i < len(data); i++ {
- if i, ok = validany(data, i); !ok {
- return i, false
- }
- if i, ok = validcomma(data, i, ']'); !ok {
- return i, false
- }
- if data[i] == ']' {
- return i + 1, true
- }
- }
- case ' ', '\t', '\n', '\r':
- continue
- case ']':
- return i + 1, true
- }
- }
- return i, false
-}
-func validstring(data []byte, i int) (outi int, ok bool) {
- for ; i < len(data); i++ {
- if data[i] < ' ' {
- return i, false
- } else if data[i] == '\\' {
- i++
- if i == len(data) {
- return i, false
- }
- switch data[i] {
- default:
- return i, false
- case '"', '\\', '/', 'b', 'f', 'n', 'r', 't':
- case 'u':
- for j := 0; j < 4; j++ {
- i++
- if i >= len(data) {
- return i, false
- }
- if !((data[i] >= '0' && data[i] <= '9') ||
- (data[i] >= 'a' && data[i] <= 'f') ||
- (data[i] >= 'A' && data[i] <= 'F')) {
- return i, false
- }
- }
- }
- } else if data[i] == '"' {
- return i + 1, true
- }
- }
- return i, false
-}
-func validnumber(data []byte, i int) (outi int, ok bool) {
- i--
- // sign
- if data[i] == '-' {
- i++
- if i == len(data) {
- return i, false
- }
- if data[i] < '0' || data[i] > '9' {
- return i, false
- }
- }
- // int
- if i == len(data) {
- return i, false
- }
- if data[i] == '0' {
- i++
- } else {
- for ; i < len(data); i++ {
- if data[i] >= '0' && data[i] <= '9' {
- continue
- }
- break
- }
- }
- // frac
- if i == len(data) {
- return i, true
- }
- if data[i] == '.' {
- i++
- if i == len(data) {
- return i, false
- }
- if data[i] < '0' || data[i] > '9' {
- return i, false
- }
- i++
- for ; i < len(data); i++ {
- if data[i] >= '0' && data[i] <= '9' {
- continue
- }
- break
- }
- }
- // exp
- if i == len(data) {
- return i, true
- }
- if data[i] == 'e' || data[i] == 'E' {
- i++
- if i == len(data) {
- return i, false
- }
- if data[i] == '+' || data[i] == '-' {
- i++
- }
- if i == len(data) {
- return i, false
- }
- if data[i] < '0' || data[i] > '9' {
- return i, false
- }
- i++
- for ; i < len(data); i++ {
- if data[i] >= '0' && data[i] <= '9' {
- continue
- }
- break
- }
- }
- return i, true
-}
-
-func validtrue(data []byte, i int) (outi int, ok bool) {
- if i+3 <= len(data) && data[i] == 'r' && data[i+1] == 'u' &&
- data[i+2] == 'e' {
- return i + 3, true
- }
- return i, false
-}
-func validfalse(data []byte, i int) (outi int, ok bool) {
- if i+4 <= len(data) && data[i] == 'a' && data[i+1] == 'l' &&
- data[i+2] == 's' && data[i+3] == 'e' {
- return i + 4, true
- }
- return i, false
-}
-func validnull(data []byte, i int) (outi int, ok bool) {
- if i+3 <= len(data) && data[i] == 'u' && data[i+1] == 'l' &&
- data[i+2] == 'l' {
- return i + 3, true
- }
- return i, false
-}
-
-// Valid returns true if the input is valid json.
-//
-// if !gjson.Valid(json) {
-// return errors.New("invalid json")
-// }
-// value := gjson.Get(json, "name.last")
-func Valid(json string) bool {
- _, ok := validpayload(stringBytes(json), 0)
- return ok
-}
-
-// ValidBytes returns true if the input is valid json.
-//
-// if !gjson.Valid(json) {
-// return errors.New("invalid json")
-// }
-// value := gjson.Get(json, "name.last")
-//
-// If working with bytes, this method preferred over ValidBytes(string(data))
-func ValidBytes(json []byte) bool {
- _, ok := validpayload(json, 0)
- return ok
-}
-
-func parseUint(s string) (n uint64, ok bool) {
- var i int
- if i == len(s) {
- return 0, false
- }
- for ; i < len(s); i++ {
- if s[i] >= '0' && s[i] <= '9' {
- n = n*10 + uint64(s[i]-'0')
- } else {
- return 0, false
- }
- }
- return n, true
-}
-
-func parseInt(s string) (n int64, ok bool) {
- var i int
- var sign bool
- if len(s) > 0 && s[0] == '-' {
- sign = true
- i++
- }
- if i == len(s) {
- return 0, false
- }
- for ; i < len(s); i++ {
- if s[i] >= '0' && s[i] <= '9' {
- n = n*10 + int64(s[i]-'0')
- } else {
- return 0, false
- }
- }
- if sign {
- return n * -1, true
- }
- return n, true
-}
-
-// safeInt validates a given JSON number
-// ensures it lies within the minimum and maximum representable JSON numbers
-func safeInt(f float64) (n int64, ok bool) {
- // https://tc39.es/ecma262/#sec-number.min_safe_integer
- // https://tc39.es/ecma262/#sec-number.max_safe_integer
- if f < -9007199254740991 || f > 9007199254740991 {
- return 0, false
- }
- return int64(f), true
-}
-
-// execStatic parses the path to find a static value.
-// The input expects that the path already starts with a '!'
-func execStatic(json, path string) (pathOut, res string, ok bool) {
- name := path[1:]
- if len(name) > 0 {
- switch name[0] {
- case '{', '[', '"', '+', '-', '0', '1', '2', '3', '4', '5', '6', '7',
- '8', '9':
- _, res = parseSquash(name, 0)
- pathOut = name[len(res):]
- return pathOut, res, true
- }
- }
- for i := 1; i < len(path); i++ {
- if path[i] == '|' {
- pathOut = path[i:]
- name = path[1:i]
- break
- }
- if path[i] == '.' {
- pathOut = path[i:]
- name = path[1:i]
- break
- }
- }
- switch strings.ToLower(name) {
- case "true", "false", "null", "nan", "inf":
- return pathOut, name, true
- }
- return pathOut, res, false
-}
-
-// execModifier parses the path to find a matching modifier function.
-// The input expects that the path already starts with a '@'
-func execModifier(json, path string) (pathOut, res string, ok bool) {
- name := path[1:]
- var hasArgs bool
- for i := 1; i < len(path); i++ {
- if path[i] == ':' {
- pathOut = path[i+1:]
- name = path[1:i]
- hasArgs = len(pathOut) > 0
- break
- }
- if path[i] == '|' {
- pathOut = path[i:]
- name = path[1:i]
- break
- }
- if path[i] == '.' {
- pathOut = path[i:]
- name = path[1:i]
- break
- }
- }
- if fn, ok := modifiers[name]; ok {
- var args string
- if hasArgs {
- var parsedArgs bool
- switch pathOut[0] {
- case '{', '[', '"':
- // json arg
- res := Parse(pathOut)
- if res.Exists() {
- args = squash(pathOut)
- pathOut = pathOut[len(args):]
- parsedArgs = true
- }
- }
- if !parsedArgs {
- // simple arg
- i := 0
- for ; i < len(pathOut); i++ {
- if pathOut[i] == '|' {
- break
- }
- switch pathOut[i] {
- case '{', '[', '"', '(':
- s := squash(pathOut[i:])
- i += len(s) - 1
- }
- }
- args = pathOut[:i]
- pathOut = pathOut[i:]
- }
- }
- return pathOut, fn(json, args), true
- }
- return pathOut, res, false
-}
-
-// unwrap removes the '[]' or '{}' characters around json
-func unwrap(json string) string {
- json = trim(json)
- if len(json) >= 2 && (json[0] == '[' || json[0] == '{') {
- json = json[1 : len(json)-1]
- }
- return json
-}
-
-// DisableModifiers will disable the modifier syntax
-var DisableModifiers = false
-
-var modifiers map[string]func(json, arg string) string
-
-func init() {
- modifiers = map[string]func(json, arg string) string{
- "pretty": modPretty,
- "ugly": modUgly,
- "reverse": modReverse,
- "this": modThis,
- "flatten": modFlatten,
- "join": modJoin,
- "valid": modValid,
- "keys": modKeys,
- "values": modValues,
- "tostr": modToStr,
- "fromstr": modFromStr,
- "group": modGroup,
- "dig": modDig,
- }
-}
-
-// AddModifier binds a custom modifier command to the GJSON syntax.
-// This operation is not thread safe and should be executed prior to
-// using all other gjson function.
-func AddModifier(name string, fn func(json, arg string) string) {
- modifiers[name] = fn
-}
-
-// ModifierExists returns true when the specified modifier exists.
-func ModifierExists(name string, fn func(json, arg string) string) bool {
- _, ok := modifiers[name]
- return ok
-}
-
-// cleanWS remove any non-whitespace from string
-func cleanWS(s string) string {
- for i := 0; i < len(s); i++ {
- switch s[i] {
- case ' ', '\t', '\n', '\r':
- continue
- default:
- var s2 []byte
- for i := 0; i < len(s); i++ {
- switch s[i] {
- case ' ', '\t', '\n', '\r':
- s2 = append(s2, s[i])
- }
- }
- return string(s2)
- }
- }
- return s
-}
-
-// @pretty modifier makes the json look nice.
-func modPretty(json, arg string) string {
- if len(arg) > 0 {
- opts := *pretty.DefaultOptions
- Parse(arg).ForEach(func(key, value Result) bool {
- switch key.String() {
- case "sortKeys":
- opts.SortKeys = value.Bool()
- case "indent":
- opts.Indent = cleanWS(value.String())
- case "prefix":
- opts.Prefix = cleanWS(value.String())
- case "width":
- opts.Width = int(value.Int())
- }
- return true
- })
- return bytesString(pretty.PrettyOptions(stringBytes(json), &opts))
- }
- return bytesString(pretty.Pretty(stringBytes(json)))
-}
-
-// @this returns the current element. Can be used to retrieve the root element.
-func modThis(json, arg string) string {
- return json
-}
-
-// @ugly modifier removes all whitespace.
-func modUgly(json, arg string) string {
- return bytesString(pretty.Ugly(stringBytes(json)))
-}
-
-// @reverse reverses array elements or root object members.
-func modReverse(json, arg string) string {
- res := Parse(json)
- if res.IsArray() {
- var values []Result
- res.ForEach(func(_, value Result) bool {
- values = append(values, value)
- return true
- })
- out := make([]byte, 0, len(json))
- out = append(out, '[')
- for i, j := len(values)-1, 0; i >= 0; i, j = i-1, j+1 {
- if j > 0 {
- out = append(out, ',')
- }
- out = append(out, values[i].Raw...)
- }
- out = append(out, ']')
- return bytesString(out)
- }
- if res.IsObject() {
- var keyValues []Result
- res.ForEach(func(key, value Result) bool {
- keyValues = append(keyValues, key, value)
- return true
- })
- out := make([]byte, 0, len(json))
- out = append(out, '{')
- for i, j := len(keyValues)-2, 0; i >= 0; i, j = i-2, j+1 {
- if j > 0 {
- out = append(out, ',')
- }
- out = append(out, keyValues[i+0].Raw...)
- out = append(out, ':')
- out = append(out, keyValues[i+1].Raw...)
- }
- out = append(out, '}')
- return bytesString(out)
- }
- return json
-}
-
-// @flatten an array with child arrays.
-//
-// [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,[6,7]]
-//
-// The {"deep":true} arg can be provide for deep flattening.
-//
-// [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,6,7]
-//
-// The original json is returned when the json is not an array.
-func modFlatten(json, arg string) string {
- res := Parse(json)
- if !res.IsArray() {
- return json
- }
- var deep bool
- if arg != "" {
- Parse(arg).ForEach(func(key, value Result) bool {
- if key.String() == "deep" {
- deep = value.Bool()
- }
- return true
- })
- }
- var out []byte
- out = append(out, '[')
- var idx int
- res.ForEach(func(_, value Result) bool {
- var raw string
- if value.IsArray() {
- if deep {
- raw = unwrap(modFlatten(value.Raw, arg))
- } else {
- raw = unwrap(value.Raw)
- }
- } else {
- raw = value.Raw
- }
- raw = strings.TrimSpace(raw)
- if len(raw) > 0 {
- if idx > 0 {
- out = append(out, ',')
- }
- out = append(out, raw...)
- idx++
- }
- return true
- })
- out = append(out, ']')
- return bytesString(out)
-}
-
-// @keys extracts the keys from an object.
-//
-// {"first":"Tom","last":"Smith"} -> ["first","last"]
-func modKeys(json, arg string) string {
- v := Parse(json)
- if !v.Exists() {
- return "[]"
- }
- obj := v.IsObject()
- var out strings.Builder
- out.WriteByte('[')
- var i int
- v.ForEach(func(key, _ Result) bool {
- if i > 0 {
- out.WriteByte(',')
- }
- if obj {
- out.WriteString(key.Raw)
- } else {
- out.WriteString("null")
- }
- i++
- return true
- })
- out.WriteByte(']')
- return out.String()
-}
-
-// @values extracts the values from an object.
-//
-// {"first":"Tom","last":"Smith"} -> ["Tom","Smith"]
-func modValues(json, arg string) string {
- v := Parse(json)
- if !v.Exists() {
- return "[]"
- }
- if v.IsArray() {
- return json
- }
- var out strings.Builder
- out.WriteByte('[')
- var i int
- v.ForEach(func(_, value Result) bool {
- if i > 0 {
- out.WriteByte(',')
- }
- out.WriteString(value.Raw)
- i++
- return true
- })
- out.WriteByte(']')
- return out.String()
-}
-
-// @join multiple objects into a single object.
-//
-// [{"first":"Tom"},{"last":"Smith"}] -> {"first","Tom","last":"Smith"}
-//
-// The arg can be "true" to specify that duplicate keys should be preserved.
-//
-// [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":37,"age":41}
-//
-// Without preserved keys:
-//
-// [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":41}
-//
-// The original json is returned when the json is not an object.
-func modJoin(json, arg string) string {
- res := Parse(json)
- if !res.IsArray() {
- return json
- }
- var preserve bool
- if arg != "" {
- Parse(arg).ForEach(func(key, value Result) bool {
- if key.String() == "preserve" {
- preserve = value.Bool()
- }
- return true
- })
- }
- var out []byte
- out = append(out, '{')
- if preserve {
- // Preserve duplicate keys.
- var idx int
- res.ForEach(func(_, value Result) bool {
- if !value.IsObject() {
- return true
- }
- if idx > 0 {
- out = append(out, ',')
- }
- out = append(out, unwrap(value.Raw)...)
- idx++
- return true
- })
- } else {
- // Deduplicate keys and generate an object with stable ordering.
- var keys []Result
- kvals := make(map[string]Result)
- res.ForEach(func(_, value Result) bool {
- if !value.IsObject() {
- return true
- }
- value.ForEach(func(key, value Result) bool {
- k := key.String()
- if _, ok := kvals[k]; !ok {
- keys = append(keys, key)
- }
- kvals[k] = value
- return true
- })
- return true
- })
- for i := 0; i < len(keys); i++ {
- if i > 0 {
- out = append(out, ',')
- }
- out = append(out, keys[i].Raw...)
- out = append(out, ':')
- out = append(out, kvals[keys[i].String()].Raw...)
- }
- }
- out = append(out, '}')
- return bytesString(out)
-}
-
-// @valid ensures that the json is valid before moving on. An empty string is
-// returned when the json is not valid, otherwise it returns the original json.
-func modValid(json, arg string) string {
- if !Valid(json) {
- return ""
- }
- return json
-}
-
-// @fromstr converts a string to json
-//
-// "{\"id\":1023,\"name\":\"alert\"}" -> {"id":1023,"name":"alert"}
-func modFromStr(json, arg string) string {
- if !Valid(json) {
- return ""
- }
- return Parse(json).String()
-}
-
-// @tostr converts a string to json
-//
-// {"id":1023,"name":"alert"} -> "{\"id\":1023,\"name\":\"alert\"}"
-func modToStr(str, arg string) string {
- return string(AppendJSONString(nil, str))
-}
-
-func modGroup(json, arg string) string {
- res := Parse(json)
- if !res.IsObject() {
- return ""
- }
- var all [][]byte
- res.ForEach(func(key, value Result) bool {
- if !value.IsArray() {
- return true
- }
- var idx int
- value.ForEach(func(_, value Result) bool {
- if idx == len(all) {
- all = append(all, []byte{})
- }
- all[idx] = append(all[idx], ("," + key.Raw + ":" + value.Raw)...)
- idx++
- return true
- })
- return true
- })
- var data []byte
- data = append(data, '[')
- for i, item := range all {
- if i > 0 {
- data = append(data, ',')
- }
- data = append(data, '{')
- data = append(data, item[1:]...)
- data = append(data, '}')
- }
- data = append(data, ']')
- return string(data)
-}
-
-// stringHeader instead of reflect.StringHeader
-type stringHeader struct {
- data unsafe.Pointer
- len int
-}
-
-// sliceHeader instead of reflect.SliceHeader
-type sliceHeader struct {
- data unsafe.Pointer
- len int
- cap int
-}
-
-// getBytes casts the input json bytes to a string and safely returns the
-// results as uniquely allocated data. This operation is intended to minimize
-// copies and allocations for the large json string->[]byte.
-func getBytes(json []byte, path string) Result {
- var result Result
- if json != nil {
- // unsafe cast to string
- result = Get(*(*string)(unsafe.Pointer(&json)), path)
- // safely get the string headers
- rawhi := *(*stringHeader)(unsafe.Pointer(&result.Raw))
- strhi := *(*stringHeader)(unsafe.Pointer(&result.Str))
- // create byte slice headers
- rawh := sliceHeader{data: rawhi.data, len: rawhi.len, cap: rawhi.len}
- strh := sliceHeader{data: strhi.data, len: strhi.len, cap: rawhi.len}
- if strh.data == nil {
- // str is nil
- if rawh.data == nil {
- // raw is nil
- result.Raw = ""
- } else {
- // raw has data, safely copy the slice header to a string
- result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
- }
- result.Str = ""
- } else if rawh.data == nil {
- // raw is nil
- result.Raw = ""
- // str has data, safely copy the slice header to a string
- result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
- } else if uintptr(strh.data) >= uintptr(rawh.data) &&
- uintptr(strh.data)+uintptr(strh.len) <=
- uintptr(rawh.data)+uintptr(rawh.len) {
- // Str is a substring of Raw.
- start := uintptr(strh.data) - uintptr(rawh.data)
- // safely copy the raw slice header
- result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
- // substring the raw
- result.Str = result.Raw[start : start+uintptr(strh.len)]
- } else {
- // safely copy both the raw and str slice headers to strings
- result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
- result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
- }
- }
- return result
-}
-
-// fillIndex finds the position of Raw data and assigns it to the Index field
-// of the resulting value. If the position cannot be found then Index zero is
-// used instead.
-func fillIndex(json string, c *parseContext) {
- if len(c.value.Raw) > 0 && !c.calcd {
- jhdr := *(*stringHeader)(unsafe.Pointer(&json))
- rhdr := *(*stringHeader)(unsafe.Pointer(&(c.value.Raw)))
- c.value.Index = int(uintptr(rhdr.data) - uintptr(jhdr.data))
- if c.value.Index < 0 || c.value.Index >= len(json) {
- c.value.Index = 0
- }
- }
-}
-
-func stringBytes(s string) []byte {
- return *(*[]byte)(unsafe.Pointer(&sliceHeader{
- data: (*stringHeader)(unsafe.Pointer(&s)).data,
- len: len(s),
- cap: len(s),
- }))
-}
-
-func bytesString(b []byte) string {
- return *(*string)(unsafe.Pointer(&b))
-}
-
-func revSquash(json string) string {
- // reverse squash
- // expects that the tail character is a ']' or '}' or ')' or '"'
- // squash the value, ignoring all nested arrays and objects.
- i := len(json) - 1
- var depth int
- if json[i] != '"' {
- depth++
- }
- if json[i] == '}' || json[i] == ']' || json[i] == ')' {
- i--
- }
- for ; i >= 0; i-- {
- switch json[i] {
- case '"':
- i--
- for ; i >= 0; i-- {
- if json[i] == '"' {
- esc := 0
- for i > 0 && json[i-1] == '\\' {
- i--
- esc++
- }
- if esc%2 == 1 {
- continue
- }
- i += esc
- break
- }
- }
- if depth == 0 {
- if i < 0 {
- i = 0
- }
- return json[i:]
- }
- case '}', ']', ')':
- depth++
- case '{', '[', '(':
- depth--
- if depth == 0 {
- return json[i:]
- }
- }
- }
- return json
-}
-
-// Paths returns the original GJSON paths for a Result where the Result came
-// from a simple query path that returns an array, like:
-//
-// gjson.Get(json, "friends.#.first")
-//
-// The returned value will be in the form of a JSON array:
-//
-// ["friends.0.first","friends.1.first","friends.2.first"]
-//
-// The param 'json' must be the original JSON used when calling Get.
-//
-// Returns an empty string if the paths cannot be determined, which can happen
-// when the Result came from a path that contained a multipath, modifier,
-// or a nested query.
-func (t Result) Paths(json string) []string {
- if t.Indexes == nil {
- return nil
- }
- paths := make([]string, 0, len(t.Indexes))
- t.ForEach(func(_, value Result) bool {
- paths = append(paths, value.Path(json))
- return true
- })
- if len(paths) != len(t.Indexes) {
- return nil
- }
- return paths
-}
-
-// Path returns the original GJSON path for a Result where the Result came
-// from a simple path that returns a single value, like:
-//
-// gjson.Get(json, "friends.#(last=Murphy)")
-//
-// The returned value will be in the form of a JSON string:
-//
-// "friends.0"
-//
-// The param 'json' must be the original JSON used when calling Get.
-//
-// Returns an empty string if the paths cannot be determined, which can happen
-// when the Result came from a path that contained a multipath, modifier,
-// or a nested query.
-func (t Result) Path(json string) string {
- var path []byte
- var comps []string // raw components
- i := t.Index - 1
- if t.Index+len(t.Raw) > len(json) {
- // JSON cannot safely contain Result.
- goto fail
- }
- if !strings.HasPrefix(json[t.Index:], t.Raw) {
- // Result is not at the JSON index as expected.
- goto fail
- }
- for ; i >= 0; i-- {
- if json[i] <= ' ' {
- continue
- }
- if json[i] == ':' {
- // inside of object, get the key
- for ; i >= 0; i-- {
- if json[i] != '"' {
- continue
- }
- break
- }
- raw := revSquash(json[:i+1])
- i = i - len(raw)
- comps = append(comps, raw)
- // key gotten, now squash the rest
- raw = revSquash(json[:i+1])
- i = i - len(raw)
- i++ // increment the index for next loop step
- } else if json[i] == '{' {
- // Encountered an open object. The original result was probably an
- // object key.
- goto fail
- } else if json[i] == ',' || json[i] == '[' {
- // inside of an array, count the position
- var arrIdx int
- if json[i] == ',' {
- arrIdx++
- i--
- }
- for ; i >= 0; i-- {
- if json[i] == ':' {
- // Encountered an unexpected colon. The original result was
- // probably an object key.
- goto fail
- } else if json[i] == ',' {
- arrIdx++
- } else if json[i] == '[' {
- comps = append(comps, strconv.Itoa(arrIdx))
- break
- } else if json[i] == ']' || json[i] == '}' || json[i] == '"' {
- raw := revSquash(json[:i+1])
- i = i - len(raw) + 1
- }
- }
- }
- }
- if len(comps) == 0 {
- if DisableModifiers {
- goto fail
- }
- return "@this"
- }
- for i := len(comps) - 1; i >= 0; i-- {
- rcomp := Parse(comps[i])
- if !rcomp.Exists() {
- goto fail
- }
- comp := Escape(rcomp.String())
- path = append(path, '.')
- path = append(path, comp...)
- }
- if len(path) > 0 {
- path = path[1:]
- }
- return string(path)
-fail:
- return ""
-}
-
-// isSafePathKeyChar returns true if the input character is safe for not
-// needing escaping.
-func isSafePathKeyChar(c byte) bool {
- return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
- (c >= '0' && c <= '9') || c <= ' ' || c > '~' || c == '_' ||
- c == '-' || c == ':'
-}
-
-// Escape returns an escaped path component.
-//
-// json := `{
-// "user":{
-// "first.name": "Janet",
-// "last.name": "Prichard"
-// }
-// }`
-// user := gjson.Get(json, "user")
-// println(user.Get(gjson.Escape("first.name"))
-// println(user.Get(gjson.Escape("last.name"))
-// // Output:
-// // Janet
-// // Prichard
-func Escape(comp string) string {
- for i := 0; i < len(comp); i++ {
- if !isSafePathKeyChar(comp[i]) {
- ncomp := make([]byte, len(comp)+1)
- copy(ncomp, comp[:i])
- ncomp = ncomp[:i]
- for ; i < len(comp); i++ {
- if !isSafePathKeyChar(comp[i]) {
- ncomp = append(ncomp, '\\')
- }
- ncomp = append(ncomp, comp[i])
- }
- return string(ncomp)
- }
- }
- return comp
-}
-
-func parseRecursiveDescent(all []Result, parent Result, path string) []Result {
- if res := parent.Get(path); res.Exists() {
- all = append(all, res)
- }
- if parent.IsArray() || parent.IsObject() {
- parent.ForEach(func(_, val Result) bool {
- all = parseRecursiveDescent(all, val, path)
- return true
- })
- }
- return all
-}
-
-func modDig(json, arg string) string {
- all := parseRecursiveDescent(nil, Parse(json), arg)
- var out []byte
- out = append(out, '[')
- for i, res := range all {
- if i > 0 {
- out = append(out, ',')
- }
- out = append(out, res.Raw...)
- }
- out = append(out, ']')
- return string(out)
-}
diff --git a/vendor/github.com/tidwall/match/LICENSE b/vendor/github.com/tidwall/match/LICENSE
deleted file mode 100644
index 58f5819a..00000000
--- a/vendor/github.com/tidwall/match/LICENSE
+++ /dev/null
@@ -1,20 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2016 Josh Baker
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/vendor/github.com/tidwall/match/README.md b/vendor/github.com/tidwall/match/README.md
deleted file mode 100644
index 5fdd4cf6..00000000
--- a/vendor/github.com/tidwall/match/README.md
+++ /dev/null
@@ -1,29 +0,0 @@
-# Match
-
-[](https://godoc.org/github.com/tidwall/match)
-
-Match is a very simple pattern matcher where '*' matches on any
-number characters and '?' matches on any one character.
-
-## Installing
-
-```
-go get -u github.com/tidwall/match
-```
-
-## Example
-
-```go
-match.Match("hello", "*llo")
-match.Match("jello", "?ello")
-match.Match("hello", "h*o")
-```
-
-
-## Contact
-
-Josh Baker [@tidwall](http://twitter.com/tidwall)
-
-## License
-
-Redcon source code is available under the MIT [License](/LICENSE).
diff --git a/vendor/github.com/tidwall/match/match.go b/vendor/github.com/tidwall/match/match.go
deleted file mode 100644
index 11da28f1..00000000
--- a/vendor/github.com/tidwall/match/match.go
+++ /dev/null
@@ -1,237 +0,0 @@
-// Package match provides a simple pattern matcher with unicode support.
-package match
-
-import (
- "unicode/utf8"
-)
-
-// Match returns true if str matches pattern. This is a very
-// simple wildcard match where '*' matches on any number characters
-// and '?' matches on any one character.
-//
-// pattern:
-// { term }
-// term:
-// '*' matches any sequence of non-Separator characters
-// '?' matches any single non-Separator character
-// c matches character c (c != '*', '?', '\\')
-// '\\' c matches character c
-//
-func Match(str, pattern string) bool {
- if pattern == "*" {
- return true
- }
- return match(str, pattern, 0, nil, -1) == rMatch
-}
-
-// MatchLimit is the same as Match but will limit the complexity of the match
-// operation. This is to avoid long running matches, specifically to avoid ReDos
-// attacks from arbritary inputs.
-//
-// How it works:
-// The underlying match routine is recursive and may call itself when it
-// encounters a sandwiched wildcard pattern, such as: `user:*:name`.
-// Everytime it calls itself a counter is incremented.
-// The operation is stopped when counter > maxcomp*len(str).
-func MatchLimit(str, pattern string, maxcomp int) (matched, stopped bool) {
- if pattern == "*" {
- return true, false
- }
- counter := 0
- r := match(str, pattern, len(str), &counter, maxcomp)
- if r == rStop {
- return false, true
- }
- return r == rMatch, false
-}
-
-type result int
-
-const (
- rNoMatch result = iota
- rMatch
- rStop
-)
-
-func match(str, pat string, slen int, counter *int, maxcomp int) result {
- // check complexity limit
- if maxcomp > -1 {
- if *counter > slen*maxcomp {
- return rStop
- }
- *counter++
- }
-
- for len(pat) > 0 {
- var wild bool
- pc, ps := rune(pat[0]), 1
- if pc > 0x7f {
- pc, ps = utf8.DecodeRuneInString(pat)
- }
- var sc rune
- var ss int
- if len(str) > 0 {
- sc, ss = rune(str[0]), 1
- if sc > 0x7f {
- sc, ss = utf8.DecodeRuneInString(str)
- }
- }
- switch pc {
- case '?':
- if ss == 0 {
- return rNoMatch
- }
- case '*':
- // Ignore repeating stars.
- for len(pat) > 1 && pat[1] == '*' {
- pat = pat[1:]
- }
-
- // If this star is the last character then it must be a match.
- if len(pat) == 1 {
- return rMatch
- }
-
- // Match and trim any non-wildcard suffix characters.
- var ok bool
- str, pat, ok = matchTrimSuffix(str, pat)
- if !ok {
- return rNoMatch
- }
-
- // Check for single star again.
- if len(pat) == 1 {
- return rMatch
- }
-
- // Perform recursive wildcard search.
- r := match(str, pat[1:], slen, counter, maxcomp)
- if r != rNoMatch {
- return r
- }
- if len(str) == 0 {
- return rNoMatch
- }
- wild = true
- default:
- if ss == 0 {
- return rNoMatch
- }
- if pc == '\\' {
- pat = pat[ps:]
- pc, ps = utf8.DecodeRuneInString(pat)
- if ps == 0 {
- return rNoMatch
- }
- }
- if sc != pc {
- return rNoMatch
- }
- }
- str = str[ss:]
- if !wild {
- pat = pat[ps:]
- }
- }
- if len(str) == 0 {
- return rMatch
- }
- return rNoMatch
-}
-
-// matchTrimSuffix matches and trims any non-wildcard suffix characters.
-// Returns the trimed string and pattern.
-//
-// This is called because the pattern contains extra data after the wildcard
-// star. Here we compare any suffix characters in the pattern to the suffix of
-// the target string. Basically a reverse match that stops when a wildcard
-// character is reached. This is a little trickier than a forward match because
-// we need to evaluate an escaped character in reverse.
-//
-// Any matched characters will be trimmed from both the target
-// string and the pattern.
-func matchTrimSuffix(str, pat string) (string, string, bool) {
- // It's expected that the pattern has at least two bytes and the first byte
- // is a wildcard star '*'
- match := true
- for len(str) > 0 && len(pat) > 1 {
- pc, ps := utf8.DecodeLastRuneInString(pat)
- var esc bool
- for i := 0; ; i++ {
- if pat[len(pat)-ps-i-1] != '\\' {
- if i&1 == 1 {
- esc = true
- ps++
- }
- break
- }
- }
- if pc == '*' && !esc {
- match = true
- break
- }
- sc, ss := utf8.DecodeLastRuneInString(str)
- if !((pc == '?' && !esc) || pc == sc) {
- match = false
- break
- }
- str = str[:len(str)-ss]
- pat = pat[:len(pat)-ps]
- }
- return str, pat, match
-}
-
-var maxRuneBytes = [...]byte{244, 143, 191, 191}
-
-// Allowable parses the pattern and determines the minimum and maximum allowable
-// values that the pattern can represent.
-// When the max cannot be determined, 'true' will be returned
-// for infinite.
-func Allowable(pattern string) (min, max string) {
- if pattern == "" || pattern[0] == '*' {
- return "", ""
- }
-
- minb := make([]byte, 0, len(pattern))
- maxb := make([]byte, 0, len(pattern))
- var wild bool
- for i := 0; i < len(pattern); i++ {
- if pattern[i] == '*' {
- wild = true
- break
- }
- if pattern[i] == '?' {
- minb = append(minb, 0)
- maxb = append(maxb, maxRuneBytes[:]...)
- } else {
- minb = append(minb, pattern[i])
- maxb = append(maxb, pattern[i])
- }
- }
- if wild {
- r, n := utf8.DecodeLastRune(maxb)
- if r != utf8.RuneError {
- if r < utf8.MaxRune {
- r++
- if r > 0x7f {
- b := make([]byte, 4)
- nn := utf8.EncodeRune(b, r)
- maxb = append(maxb[:len(maxb)-n], b[:nn]...)
- } else {
- maxb = append(maxb[:len(maxb)-n], byte(r))
- }
- }
- }
- }
- return string(minb), string(maxb)
-}
-
-// IsPattern returns true if the string is a pattern.
-func IsPattern(str string) bool {
- for i := 0; i < len(str); i++ {
- if str[i] == '*' || str[i] == '?' {
- return true
- }
- }
- return false
-}
diff --git a/vendor/github.com/tidwall/pretty/LICENSE b/vendor/github.com/tidwall/pretty/LICENSE
deleted file mode 100644
index 993b83f2..00000000
--- a/vendor/github.com/tidwall/pretty/LICENSE
+++ /dev/null
@@ -1,20 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2017 Josh Baker
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/vendor/github.com/tidwall/pretty/README.md b/vendor/github.com/tidwall/pretty/README.md
deleted file mode 100644
index 76c06a5e..00000000
--- a/vendor/github.com/tidwall/pretty/README.md
+++ /dev/null
@@ -1,122 +0,0 @@
-# Pretty
-
-[](https://pkg.go.dev/github.com/tidwall/pretty)
-
-Pretty is a Go package that provides [fast](#performance) methods for formatting JSON for human readability, or to compact JSON for smaller payloads.
-
-Getting Started
-===============
-
-## Installing
-
-To start using Pretty, install Go and run `go get`:
-
-```sh
-$ go get -u github.com/tidwall/pretty
-```
-
-This will retrieve the library.
-
-## Pretty
-
-Using this example:
-
-```json
-{"name": {"first":"Tom","last":"Anderson"}, "age":37,
-"children": ["Sara","Alex","Jack"],
-"fav.movie": "Deer Hunter", "friends": [
- {"first": "Janet", "last": "Murphy", "age": 44}
- ]}
-```
-
-The following code:
-```go
-result = pretty.Pretty(example)
-```
-
-Will format the json to:
-
-```json
-{
- "name": {
- "first": "Tom",
- "last": "Anderson"
- },
- "age": 37,
- "children": ["Sara", "Alex", "Jack"],
- "fav.movie": "Deer Hunter",
- "friends": [
- {
- "first": "Janet",
- "last": "Murphy",
- "age": 44
- }
- ]
-}
-```
-
-## Color
-
-Color will colorize the json for outputing to the screen.
-
-```go
-result = pretty.Color(json, nil)
-```
-
-Will add color to the result for printing to the terminal.
-The second param is used for a customizing the style, and passing nil will use the default `pretty.TerminalStyle`.
-
-## Ugly
-
-The following code:
-```go
-result = pretty.Ugly(example)
-```
-
-Will format the json to:
-
-```json
-{"name":{"first":"Tom","last":"Anderson"},"age":37,"children":["Sara","Alex","Jack"],"fav.movie":"Deer Hunter","friends":[{"first":"Janet","last":"Murphy","age":44}]}```
-```
-
-## Customized output
-
-There's a `PrettyOptions(json, opts)` function which allows for customizing the output with the following options:
-
-```go
-type Options struct {
- // Width is an max column width for single line arrays
- // Default is 80
- Width int
- // Prefix is a prefix for all lines
- // Default is an empty string
- Prefix string
- // Indent is the nested indentation
- // Default is two spaces
- Indent string
- // SortKeys will sort the keys alphabetically
- // Default is false
- SortKeys bool
-}
-```
-## Performance
-
-Benchmarks of Pretty alongside the builtin `encoding/json` Indent/Compact methods.
-```
-BenchmarkPretty-16 1000000 1034 ns/op 720 B/op 2 allocs/op
-BenchmarkPrettySortKeys-16 586797 1983 ns/op 2848 B/op 14 allocs/op
-BenchmarkUgly-16 4652365 254 ns/op 240 B/op 1 allocs/op
-BenchmarkUglyInPlace-16 6481233 183 ns/op 0 B/op 0 allocs/op
-BenchmarkJSONIndent-16 450654 2687 ns/op 1221 B/op 0 allocs/op
-BenchmarkJSONCompact-16 685111 1699 ns/op 442 B/op 0 allocs/op
-```
-
-*These benchmarks were run on a MacBook Pro 2.4 GHz 8-Core Intel Core i9.*
-
-## Contact
-Josh Baker [@tidwall](http://twitter.com/tidwall)
-
-## License
-
-Pretty source code is available under the MIT [License](/LICENSE).
-
diff --git a/vendor/github.com/tidwall/pretty/pretty.go b/vendor/github.com/tidwall/pretty/pretty.go
deleted file mode 100644
index d705f9cd..00000000
--- a/vendor/github.com/tidwall/pretty/pretty.go
+++ /dev/null
@@ -1,682 +0,0 @@
-package pretty
-
-import (
- "bytes"
- "encoding/json"
- "sort"
- "strconv"
-)
-
-// Options is Pretty options
-type Options struct {
- // Width is an max column width for single line arrays
- // Default is 80
- Width int
- // Prefix is a prefix for all lines
- // Default is an empty string
- Prefix string
- // Indent is the nested indentation
- // Default is two spaces
- Indent string
- // SortKeys will sort the keys alphabetically
- // Default is false
- SortKeys bool
-}
-
-// DefaultOptions is the default options for pretty formats.
-var DefaultOptions = &Options{Width: 80, Prefix: "", Indent: " ", SortKeys: false}
-
-// Pretty converts the input json into a more human readable format where each
-// element is on it's own line with clear indentation.
-func Pretty(json []byte) []byte { return PrettyOptions(json, nil) }
-
-// PrettyOptions is like Pretty but with customized options.
-func PrettyOptions(json []byte, opts *Options) []byte {
- if opts == nil {
- opts = DefaultOptions
- }
- buf := make([]byte, 0, len(json))
- if len(opts.Prefix) != 0 {
- buf = append(buf, opts.Prefix...)
- }
- buf, _, _, _ = appendPrettyAny(buf, json, 0, true,
- opts.Width, opts.Prefix, opts.Indent, opts.SortKeys,
- 0, 0, -1)
- if len(buf) > 0 {
- buf = append(buf, '\n')
- }
- return buf
-}
-
-// Ugly removes insignificant space characters from the input json byte slice
-// and returns the compacted result.
-func Ugly(json []byte) []byte {
- buf := make([]byte, 0, len(json))
- return ugly(buf, json)
-}
-
-// UglyInPlace removes insignificant space characters from the input json
-// byte slice and returns the compacted result. This method reuses the
-// input json buffer to avoid allocations. Do not use the original bytes
-// slice upon return.
-func UglyInPlace(json []byte) []byte { return ugly(json, json) }
-
-func ugly(dst, src []byte) []byte {
- dst = dst[:0]
- for i := 0; i < len(src); i++ {
- if src[i] > ' ' {
- dst = append(dst, src[i])
- if src[i] == '"' {
- for i = i + 1; i < len(src); i++ {
- dst = append(dst, src[i])
- if src[i] == '"' {
- j := i - 1
- for ; ; j-- {
- if src[j] != '\\' {
- break
- }
- }
- if (j-i)%2 != 0 {
- break
- }
- }
- }
- }
- }
- }
- return dst
-}
-
-func isNaNOrInf(src []byte) bool {
- return src[0] == 'i' || //Inf
- src[0] == 'I' || // inf
- src[0] == '+' || // +Inf
- src[0] == 'N' || // Nan
- (src[0] == 'n' && len(src) > 1 && src[1] != 'u') // nan
-}
-
-func appendPrettyAny(buf, json []byte, i int, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) {
- for ; i < len(json); i++ {
- if json[i] <= ' ' {
- continue
- }
- if json[i] == '"' {
- return appendPrettyString(buf, json, i, nl)
- }
-
- if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' || isNaNOrInf(json[i:]) {
- return appendPrettyNumber(buf, json, i, nl)
- }
- if json[i] == '{' {
- return appendPrettyObject(buf, json, i, '{', '}', pretty, width, prefix, indent, sortkeys, tabs, nl, max)
- }
- if json[i] == '[' {
- return appendPrettyObject(buf, json, i, '[', ']', pretty, width, prefix, indent, sortkeys, tabs, nl, max)
- }
- switch json[i] {
- case 't':
- return append(buf, 't', 'r', 'u', 'e'), i + 4, nl, true
- case 'f':
- return append(buf, 'f', 'a', 'l', 's', 'e'), i + 5, nl, true
- case 'n':
- return append(buf, 'n', 'u', 'l', 'l'), i + 4, nl, true
- }
- }
- return buf, i, nl, true
-}
-
-type pair struct {
- kstart, kend int
- vstart, vend int
-}
-
-type byKeyVal struct {
- sorted bool
- json []byte
- buf []byte
- pairs []pair
-}
-
-func (arr *byKeyVal) Len() int {
- return len(arr.pairs)
-}
-func (arr *byKeyVal) Less(i, j int) bool {
- if arr.isLess(i, j, byKey) {
- return true
- }
- if arr.isLess(j, i, byKey) {
- return false
- }
- return arr.isLess(i, j, byVal)
-}
-func (arr *byKeyVal) Swap(i, j int) {
- arr.pairs[i], arr.pairs[j] = arr.pairs[j], arr.pairs[i]
- arr.sorted = true
-}
-
-type byKind int
-
-const (
- byKey byKind = 0
- byVal byKind = 1
-)
-
-type jtype int
-
-const (
- jnull jtype = iota
- jfalse
- jnumber
- jstring
- jtrue
- jjson
-)
-
-func getjtype(v []byte) jtype {
- if len(v) == 0 {
- return jnull
- }
- switch v[0] {
- case '"':
- return jstring
- case 'f':
- return jfalse
- case 't':
- return jtrue
- case 'n':
- return jnull
- case '[', '{':
- return jjson
- default:
- return jnumber
- }
-}
-
-func (arr *byKeyVal) isLess(i, j int, kind byKind) bool {
- k1 := arr.json[arr.pairs[i].kstart:arr.pairs[i].kend]
- k2 := arr.json[arr.pairs[j].kstart:arr.pairs[j].kend]
- var v1, v2 []byte
- if kind == byKey {
- v1 = k1
- v2 = k2
- } else {
- v1 = bytes.TrimSpace(arr.buf[arr.pairs[i].vstart:arr.pairs[i].vend])
- v2 = bytes.TrimSpace(arr.buf[arr.pairs[j].vstart:arr.pairs[j].vend])
- if len(v1) >= len(k1)+1 {
- v1 = bytes.TrimSpace(v1[len(k1)+1:])
- }
- if len(v2) >= len(k2)+1 {
- v2 = bytes.TrimSpace(v2[len(k2)+1:])
- }
- }
- t1 := getjtype(v1)
- t2 := getjtype(v2)
- if t1 < t2 {
- return true
- }
- if t1 > t2 {
- return false
- }
- if t1 == jstring {
- s1 := parsestr(v1)
- s2 := parsestr(v2)
- return string(s1) < string(s2)
- }
- if t1 == jnumber {
- n1, _ := strconv.ParseFloat(string(v1), 64)
- n2, _ := strconv.ParseFloat(string(v2), 64)
- return n1 < n2
- }
- return string(v1) < string(v2)
-
-}
-
-func parsestr(s []byte) []byte {
- for i := 1; i < len(s); i++ {
- if s[i] == '\\' {
- var str string
- json.Unmarshal(s, &str)
- return []byte(str)
- }
- if s[i] == '"' {
- return s[1:i]
- }
- }
- return nil
-}
-
-func appendPrettyObject(buf, json []byte, i int, open, close byte, pretty bool, width int, prefix, indent string, sortkeys bool, tabs, nl, max int) ([]byte, int, int, bool) {
- var ok bool
- if width > 0 {
- if pretty && open == '[' && max == -1 {
- // here we try to create a single line array
- max := width - (len(buf) - nl)
- if max > 3 {
- s1, s2 := len(buf), i
- buf, i, _, ok = appendPrettyObject(buf, json, i, '[', ']', false, width, prefix, "", sortkeys, 0, 0, max)
- if ok && len(buf)-s1 <= max {
- return buf, i, nl, true
- }
- buf = buf[:s1]
- i = s2
- }
- } else if max != -1 && open == '{' {
- return buf, i, nl, false
- }
- }
- buf = append(buf, open)
- i++
- var pairs []pair
- if open == '{' && sortkeys {
- pairs = make([]pair, 0, 8)
- }
- var n int
- for ; i < len(json); i++ {
- if json[i] <= ' ' {
- continue
- }
- if json[i] == close {
- if pretty {
- if open == '{' && sortkeys {
- buf = sortPairs(json, buf, pairs)
- }
- if n > 0 {
- nl = len(buf)
- if buf[nl-1] == ' ' {
- buf[nl-1] = '\n'
- } else {
- buf = append(buf, '\n')
- }
- }
- if buf[len(buf)-1] != open {
- buf = appendTabs(buf, prefix, indent, tabs)
- }
- }
- buf = append(buf, close)
- return buf, i + 1, nl, open != '{'
- }
- if open == '[' || json[i] == '"' {
- if n > 0 {
- buf = append(buf, ',')
- if width != -1 && open == '[' {
- buf = append(buf, ' ')
- }
- }
- var p pair
- if pretty {
- nl = len(buf)
- if buf[nl-1] == ' ' {
- buf[nl-1] = '\n'
- } else {
- buf = append(buf, '\n')
- }
- if open == '{' && sortkeys {
- p.kstart = i
- p.vstart = len(buf)
- }
- buf = appendTabs(buf, prefix, indent, tabs+1)
- }
- if open == '{' {
- buf, i, nl, _ = appendPrettyString(buf, json, i, nl)
- if sortkeys {
- p.kend = i
- }
- buf = append(buf, ':')
- if pretty {
- buf = append(buf, ' ')
- }
- }
- buf, i, nl, ok = appendPrettyAny(buf, json, i, pretty, width, prefix, indent, sortkeys, tabs+1, nl, max)
- if max != -1 && !ok {
- return buf, i, nl, false
- }
- if pretty && open == '{' && sortkeys {
- p.vend = len(buf)
- if p.kstart > p.kend || p.vstart > p.vend {
- // bad data. disable sorting
- sortkeys = false
- } else {
- pairs = append(pairs, p)
- }
- }
- i--
- n++
- }
- }
- return buf, i, nl, open != '{'
-}
-func sortPairs(json, buf []byte, pairs []pair) []byte {
- if len(pairs) == 0 {
- return buf
- }
- vstart := pairs[0].vstart
- vend := pairs[len(pairs)-1].vend
- arr := byKeyVal{false, json, buf, pairs}
- sort.Stable(&arr)
- if !arr.sorted {
- return buf
- }
- nbuf := make([]byte, 0, vend-vstart)
- for i, p := range pairs {
- nbuf = append(nbuf, buf[p.vstart:p.vend]...)
- if i < len(pairs)-1 {
- nbuf = append(nbuf, ',')
- nbuf = append(nbuf, '\n')
- }
- }
- return append(buf[:vstart], nbuf...)
-}
-
-func appendPrettyString(buf, json []byte, i, nl int) ([]byte, int, int, bool) {
- s := i
- i++
- for ; i < len(json); i++ {
- if json[i] == '"' {
- var sc int
- for j := i - 1; j > s; j-- {
- if json[j] == '\\' {
- sc++
- } else {
- break
- }
- }
- if sc%2 == 1 {
- continue
- }
- i++
- break
- }
- }
- return append(buf, json[s:i]...), i, nl, true
-}
-
-func appendPrettyNumber(buf, json []byte, i, nl int) ([]byte, int, int, bool) {
- s := i
- i++
- for ; i < len(json); i++ {
- if json[i] <= ' ' || json[i] == ',' || json[i] == ':' || json[i] == ']' || json[i] == '}' {
- break
- }
- }
- return append(buf, json[s:i]...), i, nl, true
-}
-
-func appendTabs(buf []byte, prefix, indent string, tabs int) []byte {
- if len(prefix) != 0 {
- buf = append(buf, prefix...)
- }
- if len(indent) == 2 && indent[0] == ' ' && indent[1] == ' ' {
- for i := 0; i < tabs; i++ {
- buf = append(buf, ' ', ' ')
- }
- } else {
- for i := 0; i < tabs; i++ {
- buf = append(buf, indent...)
- }
- }
- return buf
-}
-
-// Style is the color style
-type Style struct {
- Key, String, Number [2]string
- True, False, Null [2]string
- Escape [2]string
- Brackets [2]string
- Append func(dst []byte, c byte) []byte
-}
-
-func hexp(p byte) byte {
- switch {
- case p < 10:
- return p + '0'
- default:
- return (p - 10) + 'a'
- }
-}
-
-// TerminalStyle is for terminals
-var TerminalStyle *Style
-
-func init() {
- TerminalStyle = &Style{
- Key: [2]string{"\x1B[1m\x1B[94m", "\x1B[0m"},
- String: [2]string{"\x1B[32m", "\x1B[0m"},
- Number: [2]string{"\x1B[33m", "\x1B[0m"},
- True: [2]string{"\x1B[36m", "\x1B[0m"},
- False: [2]string{"\x1B[36m", "\x1B[0m"},
- Null: [2]string{"\x1B[2m", "\x1B[0m"},
- Escape: [2]string{"\x1B[35m", "\x1B[0m"},
- Brackets: [2]string{"\x1B[1m", "\x1B[0m"},
- Append: func(dst []byte, c byte) []byte {
- if c < ' ' && (c != '\r' && c != '\n' && c != '\t' && c != '\v') {
- dst = append(dst, "\\u00"...)
- dst = append(dst, hexp((c>>4)&0xF))
- return append(dst, hexp((c)&0xF))
- }
- return append(dst, c)
- },
- }
-}
-
-// Color will colorize the json. The style parma is used for customizing
-// the colors. Passing nil to the style param will use the default
-// TerminalStyle.
-func Color(src []byte, style *Style) []byte {
- if style == nil {
- style = TerminalStyle
- }
- apnd := style.Append
- if apnd == nil {
- apnd = func(dst []byte, c byte) []byte {
- return append(dst, c)
- }
- }
- type stackt struct {
- kind byte
- key bool
- }
- var dst []byte
- var stack []stackt
- for i := 0; i < len(src); i++ {
- if src[i] == '"' {
- key := len(stack) > 0 && stack[len(stack)-1].key
- if key {
- dst = append(dst, style.Key[0]...)
- } else {
- dst = append(dst, style.String[0]...)
- }
- dst = apnd(dst, '"')
- esc := false
- uesc := 0
- for i = i + 1; i < len(src); i++ {
- if src[i] == '\\' {
- if key {
- dst = append(dst, style.Key[1]...)
- } else {
- dst = append(dst, style.String[1]...)
- }
- dst = append(dst, style.Escape[0]...)
- dst = apnd(dst, src[i])
- esc = true
- if i+1 < len(src) && src[i+1] == 'u' {
- uesc = 5
- } else {
- uesc = 1
- }
- } else if esc {
- dst = apnd(dst, src[i])
- if uesc == 1 {
- esc = false
- dst = append(dst, style.Escape[1]...)
- if key {
- dst = append(dst, style.Key[0]...)
- } else {
- dst = append(dst, style.String[0]...)
- }
- } else {
- uesc--
- }
- } else {
- dst = apnd(dst, src[i])
- }
- if src[i] == '"' {
- j := i - 1
- for ; ; j-- {
- if src[j] != '\\' {
- break
- }
- }
- if (j-i)%2 != 0 {
- break
- }
- }
- }
- if esc {
- dst = append(dst, style.Escape[1]...)
- } else if key {
- dst = append(dst, style.Key[1]...)
- } else {
- dst = append(dst, style.String[1]...)
- }
- } else if src[i] == '{' || src[i] == '[' {
- stack = append(stack, stackt{src[i], src[i] == '{'})
- dst = append(dst, style.Brackets[0]...)
- dst = apnd(dst, src[i])
- dst = append(dst, style.Brackets[1]...)
- } else if (src[i] == '}' || src[i] == ']') && len(stack) > 0 {
- stack = stack[:len(stack)-1]
- dst = append(dst, style.Brackets[0]...)
- dst = apnd(dst, src[i])
- dst = append(dst, style.Brackets[1]...)
- } else if (src[i] == ':' || src[i] == ',') && len(stack) > 0 && stack[len(stack)-1].kind == '{' {
- stack[len(stack)-1].key = !stack[len(stack)-1].key
- dst = append(dst, style.Brackets[0]...)
- dst = apnd(dst, src[i])
- dst = append(dst, style.Brackets[1]...)
- } else {
- var kind byte
- if (src[i] >= '0' && src[i] <= '9') || src[i] == '-' || isNaNOrInf(src[i:]) {
- kind = '0'
- dst = append(dst, style.Number[0]...)
- } else if src[i] == 't' {
- kind = 't'
- dst = append(dst, style.True[0]...)
- } else if src[i] == 'f' {
- kind = 'f'
- dst = append(dst, style.False[0]...)
- } else if src[i] == 'n' {
- kind = 'n'
- dst = append(dst, style.Null[0]...)
- } else {
- dst = apnd(dst, src[i])
- }
- if kind != 0 {
- for ; i < len(src); i++ {
- if src[i] <= ' ' || src[i] == ',' || src[i] == ':' || src[i] == ']' || src[i] == '}' {
- i--
- break
- }
- dst = apnd(dst, src[i])
- }
- if kind == '0' {
- dst = append(dst, style.Number[1]...)
- } else if kind == 't' {
- dst = append(dst, style.True[1]...)
- } else if kind == 'f' {
- dst = append(dst, style.False[1]...)
- } else if kind == 'n' {
- dst = append(dst, style.Null[1]...)
- }
- }
- }
- }
- return dst
-}
-
-// Spec strips out comments and trailing commas and convert the input to a
-// valid JSON per the official spec: https://tools.ietf.org/html/rfc8259
-//
-// The resulting JSON will always be the same length as the input and it will
-// include all of the same line breaks at matching offsets. This is to ensure
-// the result can be later processed by a external parser and that that
-// parser will report messages or errors with the correct offsets.
-func Spec(src []byte) []byte {
- return spec(src, nil)
-}
-
-// SpecInPlace is the same as Spec, but this method reuses the input json
-// buffer to avoid allocations. Do not use the original bytes slice upon return.
-func SpecInPlace(src []byte) []byte {
- return spec(src, src)
-}
-
-func spec(src, dst []byte) []byte {
- dst = dst[:0]
- for i := 0; i < len(src); i++ {
- if src[i] == '/' {
- if i < len(src)-1 {
- if src[i+1] == '/' {
- dst = append(dst, ' ', ' ')
- i += 2
- for ; i < len(src); i++ {
- if src[i] == '\n' {
- dst = append(dst, '\n')
- break
- } else if src[i] == '\t' || src[i] == '\r' {
- dst = append(dst, src[i])
- } else {
- dst = append(dst, ' ')
- }
- }
- continue
- }
- if src[i+1] == '*' {
- dst = append(dst, ' ', ' ')
- i += 2
- for ; i < len(src)-1; i++ {
- if src[i] == '*' && src[i+1] == '/' {
- dst = append(dst, ' ', ' ')
- i++
- break
- } else if src[i] == '\n' || src[i] == '\t' ||
- src[i] == '\r' {
- dst = append(dst, src[i])
- } else {
- dst = append(dst, ' ')
- }
- }
- continue
- }
- }
- }
- dst = append(dst, src[i])
- if src[i] == '"' {
- for i = i + 1; i < len(src); i++ {
- dst = append(dst, src[i])
- if src[i] == '"' {
- j := i - 1
- for ; ; j-- {
- if src[j] != '\\' {
- break
- }
- }
- if (j-i)%2 != 0 {
- break
- }
- }
- }
- } else if src[i] == '}' || src[i] == ']' {
- for j := len(dst) - 2; j >= 0; j-- {
- if dst[j] <= ' ' {
- continue
- }
- if dst[j] == ',' {
- dst[j] = ' '
- }
- break
- }
- }
- }
- return dst
-}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index ea770576..7d83c30b 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -726,15 +726,6 @@ github.com/testcontainers/testcontainers-go/internal/config
github.com/testcontainers/testcontainers-go/internal/core
github.com/testcontainers/testcontainers-go/internal/core/network
github.com/testcontainers/testcontainers-go/wait
-# github.com/tidwall/gjson v1.18.0
-## explicit; go 1.12
-github.com/tidwall/gjson
-# github.com/tidwall/match v1.1.1
-## explicit; go 1.15
-github.com/tidwall/match
-# github.com/tidwall/pretty v1.2.1
-## explicit; go 1.16
-github.com/tidwall/pretty
# github.com/tklauser/go-sysconf v0.3.14
## explicit; go 1.18
github.com/tklauser/go-sysconf