From 518927c395d302ec8d38a2adabe6ece4f738eee6 Mon Sep 17 00:00:00 2001 From: Michael McGuinness Date: Mon, 20 Jan 2025 13:31:48 +0000 Subject: [PATCH] Merged in feature/update (pull request #27) Query Update * baselineupdate * baseupdateplusmodelupdates * passtests * somemoresubmittesting * testinnerfunctions * readmeandinstall * cleanerstartup * readmeplusdeps * tidyatrighttime * validatetests * normalizedontvalidate * abitofzenormalizationcleanup * addunitstestforhelperfuns * normalizeactiveversiontestas --- README.md | 7 +- api/queryService/api.gen.go | 70 ++--- api/queryService/parse.go | 50 ++-- api/queryService/parse_test.go | 53 +++- api/queryService/query.go | 30 ++- api/queryService/query_test.go | 7 +- ...create_activequeryrequirements_view.up.sql | 3 +- ..._create_activecollectorqueries_view.up.sql | 3 +- ...8174833_create_fullactivequery_view.up.sql | 13 +- database/queries/query.sql | 20 +- devbox.json | 7 +- internal/database/parseuuid.go | 90 +++++-- internal/database/parseuuid_test.go | 52 ++-- internal/database/repository/query.sql.go | 143 ++++++++--- internal/database/repository/query_test.go | 96 +++++-- internal/query/create.go | 73 ++++-- internal/query/create_test.go | 86 ++++++- internal/query/createprivate_test.go | 100 ++++++- internal/query/document/sync_test.go | 12 +- internal/query/get.go | 22 +- internal/query/get_test.go | 43 ++-- internal/query/list_test.go | 12 +- internal/query/normalize.go | 105 ++++++++ internal/query/normalize_test.go | 170 ++++++++++++ internal/query/parse_test.go | 10 +- internal/query/processor/parse.go | 8 +- internal/query/processor/parse_test.go | 2 +- internal/query/processor/service.go | 47 +++- internal/query/processor/service_test.go | 93 +++++++ internal/query/queue/create.go | 20 +- internal/query/queue/create_test.go | 8 +- internal/query/queue/execute.go | 75 +++--- internal/query/queue/execute_test.go | 29 ++- internal/query/queue/executeprivate_test.go | 15 +- internal/query/queue/result.go | 2 +- internal/query/queue/resultprivate_test.go | 4 +- internal/query/queue/service_test.go | 30 ++- .../query/types/contextFull/creator_test.go | 5 +- .../query/types/contextFull/process_test.go | 11 +- internal/query/types/contextFull/service.go | 4 +- .../query/types/contextFull/updator_test.go | 12 +- .../query/types/jsonExtractor/creator_test.go | 5 +- .../query/types/jsonExtractor/process_test.go | 43 ++-- internal/query/types/jsonExtractor/service.go | 6 +- .../query/types/jsonExtractor/updator_test.go | 12 +- internal/query/update.go | 148 ++++++++++- internal/query/update_test.go | 20 +- internal/query/updateprivate_test.go | 243 +++++++++++++++++- pkg/queryService/api.gen.go | 2 +- scripts/Taskfile.yml | 1 + serviceAPIs/queryService.yaml | 1 - test/queryService/queryservice_test.go | 43 ++-- 52 files changed, 1705 insertions(+), 461 deletions(-) create mode 100644 internal/query/normalize.go create mode 100644 internal/query/normalize_test.go create mode 100644 internal/query/processor/service_test.go diff --git a/README.md b/README.md index e1cb48c7..3a50fba8 100644 --- a/README.md +++ b/README.md @@ -8,14 +8,15 @@ Using the following project as a baseline: https://github.com/golang-standards/p - Install devbox: `https://www.jetify.com/docs/devbox/installing_devbox` - Ubuntu/MacOS: `curl -fsSL https://get.jetify.com/devbox | bash` -- Run `devbox run init` to create init dependencies -- Run `devbox shell` to enter the environment -- Run `task deps:download` to download code dependencies +- Install docker: `https://docs.docker.com/engine/install/` - (IF NECESSARY) Ensure user in docker group and docker group is in sudo group ``` sudo groupadd docker sudo usermod -aG docker $USER ``` +- Run `touch .env` to create `.env` file from custom environment variables +- Run `devbox shell` to enter the environment +- Run `task fullsuite` to run all tests that ensure the current state To find new packages: `https://search.nixos.org/packages` diff --git a/api/queryService/api.gen.go b/api/queryService/api.gen.go index 6459c433..072078d0 100644 --- a/api/queryService/api.gen.go +++ b/api/queryService/api.gen.go @@ -136,7 +136,7 @@ type Query struct { LatestVersion int32 `json:"latest_version"` // RequiredQueries List of required query IDs. - RequiredQueries []string `json:"required_queries"` + RequiredQueries *[]string `json:"required_queries,omitempty"` // Type Specifies the type of the query. Type QueryType `json:"type"` @@ -437,40 +437,40 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/9RaUW/bOBL+KwTvHn12ut0nv3WTtHDR2/RS97DAojBocmQzS5MKSbn1Bv7vC5KSTEmU", - "LTdJkT61lsgZzsw33wxHecBUbXIlQVqDpw/Y0DVsiP/v9bdcaXsFlnDhHzAwVPPcciXxFH8kO6EIQ5nS", - "CPxSZDVfrUAjDSZX0sAYj3CuVQ7acvAi7tRywVlX2HwN6E4tEWdIgyCWbwFZhewaStlOlN3lgKfYWM3l", - "Cu9HWBU2L+xCKEqCnJTY6i3iEn1dc7qOpKK/eY4yLgB95UKgJaBMFZI5ZfCNbHLh9b2eTiYPy4L+BXY/", - "eaCCg7Sc7ScPd2rp/7V8A8aSTb5fPATBnO3Hf/M8dWhjiS28M/6tIcNT/K/JIQST0v+T4PxPYe1+P8Ia", - "7guugeHpn5gzXMv5UqtQyzug1qlobE46JVfG8KWo/eB87wSC8bbLYuP0uHMJsODUcbnItVppMAaPcEa4", - "ABYpP9gXlM8DFI7DpsQLlytEZBToJmYyDoItMi4s6IQ5b/0LH1RDVQ5oSQwwpCTyG1EACdoSUQTruIXN", - "Sf+/dXuDaGdUaSXRmuzcby5XYNwBvudc9WaUE0024PZ3zQbJFoxY6EE1MRa510hlB4HjHsBp2yuKG5Rx", - "PVDYPgG2ASmtMu+G+wL0zpFDIaxx6b2s8AcsrSwGfaknhfc4WD1x8BhDVIliIzuupkoyXjHIQFRc1nv2", - "oxKikmx6ouXeVF4IqOTSuaM/aAGuaWnhHSoMZIWoaDLgsIHvjtAmilv+jWwYRR6pj3LC8ZexD48QTjA/", - "q4NSa2owjwBjFnZNnP6VBmJBVz+pUAbYgksLeksEHmGVg4x/C8jsortM89U69ZxLKgoGnvPD/1K0NmP/", - "BWPIyke4CZ8U+j9Lfl8A4szVioyDDnVSWm53p7Heg/P3anmphABqVQLo8Vu0UQxEglRCVVuUT3ddKddh", - "BapW+HPHogfSQrzl0kfweC2gbk3I0ra2JzchnOc7DPmcs5OGFG7NDzIknGegIR+4sf8rQJfqm4e5P7xo", - "nsHtctRVLhhcQJ2m3UnSqdSm0B4kdE5KqOsOF1vQppdtwhpUrmnUH2dApvSGWDzFXNrXvxy850hhFQo+", - "VTLjq670S/+80KGl9E1MLLlDG8O54bgc4Xoze9zqsOYxVlehWZyEQ7WyLOqzK3NO6al+D4DQ3C1MNsBe", - "xKiNh46rElb1ou1AVJ0G4Qw0SPh6JJI/i4/97l5PzcHYW/AtTNddTNFiA9Im+8LZVYXMaplvYRxyyYpw", - "adJ3Pe+Dfvz/PwF6J7cwEOLiei25GpQHLUfE1rTPccI/4RrcdZBvqbpG3PrWuGUDJB3SOmMQ2H+aEglN", - "dZ9yoI5/jFfndnY4o+rH3n+6+X1x/cf89s3l/OYWj/Dlze/z6z/mi7efP3xI9kte76FensfhYR9r8fgj", - "qbsSSs+k8NMJW0kWT5K43dq99zfOTHU1v/k4q9OoaY/3P7rRdA3GlrYa0FtOQxG33PrpRmrdm48z1/dX", - "0cGvxhfjCz9wyUGSnOMpfj2+GL92LQ2xa2/UJNzkfKxVIIVW3ktuuaPmeP6Sa0XB+AM5fHj1M+bqWRgN", - "hGFCyeFg7G+K7UpCtiC9FpLngocRz+TOBDQFdhs2Y6kmFftmTlldgH8Qctjb+MvFqydTfrhPeMXtxi+e", - "pwFDpqDOT1khxG7sIvHrxUXKxVsi/AjNOwstFatW/5q8LyCpbDXycqcwxWZDXNNV+f8wl3GQISvjyKAc", - "LwUw4S9uYxn9yQNne6dpBQkEXK6B/hXCTwutHfP7mZNLmcb8pwmFwzQLPOCqkQme/pnqgg715SCQu5cO", - "qniEwy09tBDNcI+i0LXp9ksHChdPjMNq0NoPhzUxaAkgUT2X6w/u9WGu1xtjH48qE30kxieifKeW/6HV", - "5cP053roowwivhdyh6DxPagZ37C4cbd9nnxPXElfUtK/j90U7sOPzfxmuL1ERJrxiCIe++dY3I9n+S1Y", - "zWHro29Ch0GbKtFyh7g1aHbVBcM7sPExftvN2HlJ39RkFdLleV4+DzRS4CRAWOCL4/x+WN7LAu/AtjHh", - "AjS7OomMEc4L29cNmUDp3LfdLfFfuV17ZqhtaMMgyGhxwmNQUNSTkifCwPPSU9k4D6Knk6Evyvb0uZqI", - "ASAL9nwX90Rd90m+Ec1hVfwpyH+MoZpb0Jx0ARcPx54xxWM1iQx/07ZgeJjiT0l9wQq9fm+Q/NzhvnZC", - "FRq/q5H1A2p+uPn4RHf5mGu15QxYf8aH7WHm9zz5FU94XlLdD1F5xnpfx6M/qFGi1dWdgesxu7qvINdA", - "y1jXFT4E/Ehlr7dVMT6DzutxDquEPGs1T8Wn1vw8PHoiNWvnIXIqlKMz2rKTQXsHnq525/dhdch+nv6r", - "/GTRl6HNhuvHUHJozuowNZqyLi8TS9fD+rGIngf0YY/J2J+g8YpHlYMKQy9JPEmnlW6cziTwia0m88li", - "ff0NaBEY3E/edeFH56S3bjf/UqU1q4PQ1HwvRKoR98sGSPzFYzhKnlp/+UUhQVJzH0f/BeFxkPOCTgPO", - "7QG9TUf6o1asoPXQGVyrX2iBp3htbW6mkwnJ+bj8Aj6majPZvsIufKW2trybCnAm/H0gMAecQ69awqVx", - "xv1omJTGlSQSlrqSDJUZhmqRsOY0bf9l/08AAAD//yOojal5KQAA", + "H4sIAAAAAAAC/9RaX2/bOBL/KgTvHn12ut0nv3WTtHDR2/RS97DAojBocmQzS5MKSbn1Bv7uC5KSTEmU", + "LTdJkT61lsgZzvx+84ejPGCqNrmSIK3B0wds6Bo2xP/3+luutL0CS7jwDxgYqnluuZJ4ij+SnVCEoUxp", + "BH4pspqvVqCRBpMraWCMRzjXKgdtOXgRd2q54KwrbL4GdKeWiDOkQRDLt4CsQnYNpWwnyu5ywFNsrOZy", + "hfcjrAqbF3YhFCVBTkps9RZxib6uOV1HUtHfPEcZF4C+ciHQElCmCsmcMvhGNrnw+l5PJ5OHZUH/Aruf", + "PFDBQVrO9pOHO7X0/1q+AWPJJt8vHoJgzvbjv3meOrSxxBbeGf/WkOEp/tfkAMGk9P8kOP9TWLvfj7CG", + "+4JrYHj6J+YM13K+1CrU8g6odSoam5NOyZUxfClqPzjfO4FgvO2y2Dg97lwCLDh1XC5yrVYajMEjnBEu", + "gEXKD/YF5fNAheO0KfnC5QoRGQHd5EzGQbBFxoUFnTDnrX/hQTVU5YCWxABDSiK/EQWSoC0RRbCOW9ic", + "9P9btzeIdkaVVhKtyc795nIFxh3ge85Vb0Y50WQDbn/XbJBswYiFHlYTY5F7jVR2EDjuIZy2vaK4QRnX", + "A4XtE2QbENIq8264L0DvXHIohDUuvJcV/4CllcWkL/Wk+B6D1YOD5xiiShQb2XE1VZLxKoMMZMVlvWc/", + "KikqyaYHLfem8kJgJZfOHf2gBbqmpYV3qDCQFaJKk4GHDX53hDZZ3PJvZMMo8kh9lBOOv4x9eCThBPOz", + "GpRaUyPzCDBmYdfE6V9pIBZ09ZMKZYAtuLSgt0TgEVY5yPi3gMwuuss0X61Tz7mkomDgc374Xyqtzdh/", + "wRiy8gg36ZNi/2fJ7wtAnLlakXHQoU5Ky+3uNNd7eP5eLS+VEECtShA9fos2ioFIJJVQ1Rbl011XynVY", + "gaoV/tyx6IFpId5y6RE8XguoWxOitK3tyU0I5/kOQz7n7KQhhVvzgwwJ5xloyAdu7P8K0KX65mHuDy+a", + "Z3C7XOoqFwwuoE7T7mTSqdSm2B4kdE5KqOsOF1vQpjfbhDWoXNOoP86ATOkNsXiKubSvfzl4zyWFVSj4", + "VMmMr7rSL/3zQoeW0jcxseRO2hieG47LEa43s8etDmseY3UFzeIkHaqVZVGfXZlzSk/1ewCF5m5hsgH2", + "IkZtPnRc1UutQ1bqdANnQC/h6xHYfhaH+t29npqDsbfg+5Wuu5iixQakTTaBs6uKhtUy3684mpIV4dKk", + "L3beB/1k/3+C4U5uYSDg4horuRpE+pYjYmva5zjhn3Dn7TrI909dI259H9yyAZIOaZ0xCOw/TcmEprpP", + "OVCXbIxX53Z2EkTVfL3/dPP74vqP+e2by/nNLR7hy5vf59d/zBdvP3/4kGyOvN5DcTwvYYd9rJW0H5mn", + "K6H0zHx9OmAryeJJArdbqPf+epmpruY3H2d1GDXt8f5HN5quwdjSVgN6y2mo2JZbP8pIrXvzceaa/Aod", + "/Gp8Mb7w05UcJMk5nuLX44vxa9e/ELv2Rk3Ctc1jrUJSaMW95Ja7PBwPW3KtKBh/IMcPr37GXPEKc4Aw", + "OcABBDD2N8V2ZUK2IL0WkueCh3nO5M4ENoXsNmygUo0l9s2YsroA/yDEsLfxl4tXT6b8cHnwittdXjw8", + "A4ZMQZ2fskKI3dgh8evFRcrFWyL8vMw7Cy0Vq1b/mrwcIKlsNd9ypzDFZkNch1X5/zCEcZQhK+OSQTlL", + "CmTCX9zGEv3JA2d7p2kFCQZcroH+FeCnhdYu8/sBkwuZxrCnSYXD6Ao84ar5CJ7+mWp5DvXlIJC7l46q", + "eITDlTz0C024RxF07XT7pUOFiyfmYTVV7afDmhi0BJCoHsL1g3t9GOL1YuzxqCLRIzE+gfKdWv6HVjcN", + "0x/roY8yiPheyB2CxpeeJr5hceMi+zzxnrh/vqSgfx+7KVx+Hxv5Tbi9RESaeESIx/45hvvxKL8Fqzls", + "PfomdBi0qRItd4hbg2ZXXTK8Axsf47fdjJ0X9E1NViFdnufl54FGCJwkCAv54nh+PyzvzQLvwLY54QCa", + "XZ1kxgjnhe3rhkxI6dy33S3xX7ld+8xQ29CmQZDRygmPYUFRj0WeiAPPm57KxnlQejoJfVG2p8/VRAwg", + "WbDnu3JP1HWfzDeiOZmKv/v4Ly9Ucwuaky7h4knYM4Z4rCYR4W/aFgyHKf5u1AdW6PV7QfJzh/vaCRU0", + "flcj6gfU/HDz8YHu4jHXassZsP6ID9vDgO954iue8Lykuh9QecZ6X+PRD2oUaHV1Z+B6zK7uK8g10BLr", + "usIHwI9U9npbhfEZ6bwe57BKyLNW8xQ+tebnyaMnQrN2HiKnoByd0ZadBO0d+HS1O78PqyH7efqv8vtE", + "X4Q2G64fk5JDc1bD1GjKunmZWLoe1o9F6XlAH/aYiP0JGq94VDmoMPQmiSfptNKN05kJfGKryXyyWF9/", + "A1qEDO4n77rwo3PSW7ebf5bSmtVBaGq+lyLViPtlEyT+4jGcJU+tv/yikEhSc4+j/4LwOMp5QacJ5/aA", + "3qaR/qgVK2g9dAbX6hda4CleW5ub6WRCcj4uP3ePqdpMtq+wg6/U1pZ3UxHOhD8GBOaIc+hVS7o0zrgf", + "DZPSuJJEwlJXkqEyw1AtEtacpu2/7P8JAAD//02yIGRmKQAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/api/queryService/parse.go b/api/queryService/parse.go index 148e4ca5..b2992bd9 100644 --- a/api/queryService/parse.go +++ b/api/queryService/parse.go @@ -4,6 +4,8 @@ import ( "errors" "queryorchestration/internal/query" queryprocessor "queryorchestration/internal/query/processor" + + "github.com/google/uuid" ) func parseQueries(queries []*query.Query) ([]Query, error) { @@ -21,9 +23,14 @@ func parseQueries(queries []*query.Query) ([]Query, error) { } func parseQuery(query *query.Query) (*Query, error) { - requiredQueries := make([]string, len(query.RequiredQueryIDs)) - for index, id := range query.RequiredQueryIDs { - requiredQueries[index] = id.String() + var requiredQueries *[]string + if query.RequiredQueryIDs != nil && len(*query.RequiredQueryIDs) > 0 { + rQ := make([]string, len(*query.RequiredQueryIDs)) + for index, id := range *query.RequiredQueryIDs { + rQ[index] = id.String() + } + + requiredQueries = &rQ } qt, err := parseQueryType(query.Type) @@ -32,18 +39,12 @@ func parseQuery(query *query.Query) (*Query, error) { } q := &Query{ - Id: query.ID.String(), - Type: qt, - ActiveVersion: query.ActiveVersion, - LatestVersion: query.LatestVersion, - } - - if query.Config != "" { - q.Config = &query.Config - } - - if len(query.RequiredQueryIDs) > 0 { - q.RequiredQueries = requiredQueries + Id: query.ID.String(), + Type: qt, + ActiveVersion: query.ActiveVersion, + LatestVersion: query.LatestVersion, + RequiredQueries: requiredQueries, + Config: query.Config, } return q, nil @@ -70,3 +71,22 @@ func parseSpecQueryType(qType QueryType) (queryprocessor.Type, error) { return queryprocessor.Type(-1), errors.New("invalid query type") } } + +func parseStringToUUIDArray(sids *[]string) (*[]uuid.UUID, error) { + var ids []uuid.UUID + if sids != nil { + ids = make([]uuid.UUID, len(*sids)) + for index, id := range *sids { + parsedID, err := uuid.Parse(id) + if err != nil { + return nil, errors.New("invalid required id") + } + + ids[index] = parsedID + } + + return &ids, nil + } + + return nil, nil +} diff --git a/api/queryService/parse_test.go b/api/queryService/parse_test.go index 27be7f2a..a54dddbc 100644 --- a/api/queryService/parse_test.go +++ b/api/queryService/parse_test.go @@ -10,16 +10,17 @@ import ( ) func TestParseQueries(t *testing.T) { + cfg := "hey" in := []*query.Query{ { ID: uuid.New(), Type: queryprocessor.TypeContextFull, ActiveVersion: 1, LatestVersion: 2, - RequiredQueryIDs: []uuid.UUID{ + RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, - Config: "hey", + Config: &cfg, }, } out, err := parseQueries(in) @@ -31,24 +32,25 @@ func TestParseQueries(t *testing.T) { Type: CONTEXTFULL, ActiveVersion: 1, LatestVersion: 2, - RequiredQueries: []string{ - in[0].RequiredQueryIDs[0].String(), + RequiredQueries: &[]string{ + (*in[0].RequiredQueryIDs)[0].String(), }, - Config: &in[0].Config, + Config: in[0].Config, }, }, out) } func TestParseQuery(t *testing.T) { + cfg := "hey" in := &query.Query{ ID: uuid.New(), Type: queryprocessor.TypeContextFull, ActiveVersion: 1, LatestVersion: 2, - RequiredQueryIDs: []uuid.UUID{ + RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, - Config: "hey", + Config: &cfg, } out, err := parseQuery(in) assert.Nil(t, err) @@ -57,10 +59,28 @@ func TestParseQuery(t *testing.T) { Type: CONTEXTFULL, ActiveVersion: 1, LatestVersion: 2, - RequiredQueries: []string{ - in.RequiredQueryIDs[0].String(), + RequiredQueries: &[]string{ + (*in.RequiredQueryIDs)[0].String(), }, - Config: &in.Config, + Config: in.Config, + }, + *out) +} + +func TestParseQueryMinimal(t *testing.T) { + in := &query.Query{ + ID: uuid.New(), + Type: queryprocessor.TypeContextFull, + ActiveVersion: 1, + LatestVersion: 2, + } + out, err := parseQuery(in) + assert.Nil(t, err) + assert.EqualExportedValues(t, Query{ + Id: in.ID.String(), + Type: CONTEXTFULL, + ActiveVersion: 1, + LatestVersion: 2, }, *out) } @@ -90,3 +110,16 @@ func TestParseSpecQueryType(t *testing.T) { _, err = parseSpecQueryType("invalid") assert.NotNil(t, err) } + +func TestParseStringToUUIDArray(t *testing.T) { + ids := []uuid.UUID{uuid.New()} + + out, err := parseStringToUUIDArray(&[]string{ids[0].String()}) + assert.Nil(t, err) + assert.ElementsMatch(t, ids, *out) + _, err = parseStringToUUIDArray(&[]string{"invalid_uuid"}) + assert.Error(t, err) + out, err = parseStringToUUIDArray(nil) + assert.Nil(t, err) + assert.Nil(t, out) +} diff --git a/api/queryService/query.go b/api/queryService/query.go index 81d894df..a056aa1b 100644 --- a/api/queryService/query.go +++ b/api/queryService/query.go @@ -53,17 +53,9 @@ func (s *Controllers) CreateQuery(ctx echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, err) } - requiredQueryIDs := []uuid.UUID{} - if req.RequiredQueries != nil { - requiredQueryIDs := make([]uuid.UUID, len(*req.RequiredQueries)) - for index, id := range *req.RequiredQueries { - parsedID, err := uuid.Parse(id) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, "Invalid Required ID") - } - - requiredQueryIDs[index] = parsedID - } + requiredQueryIDs, err := parseStringToUUIDArray(req.RequiredQueries) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, "Invalid Required IDs") } qt, err := parseSpecQueryType(req.Type) @@ -74,6 +66,7 @@ func (s *Controllers) CreateQuery(ctx echo.Context) error { id, err := s.svc.Query.Create(ctx.Request().Context(), &queryprocessor.Create{ Type: qt, RequiredQueryIDs: requiredQueryIDs, + Config: req.Config, }) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to create query: %s", err)) @@ -85,13 +78,26 @@ func (s *Controllers) CreateQuery(ctx echo.Context) error { } func (s *Controllers) UpdateQuery(ctx echo.Context, id string) error { + req := QueryUpdate{} + if err := ctx.Bind(&req); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, err) + } + uid, err := uuid.Parse(id) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, "Invalid ID") } + requiredQueryIDs, err := parseStringToUUIDArray(req.RequiredQueries) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, "Invalid Required IDs") + } + err = s.svc.Query.Update(ctx.Request().Context(), &queryprocessor.Update{ - ID: uid, + ActiveVersion: req.ActiveVersion, + Config: req.Config, + RequiredQueryIDs: requiredQueryIDs, + ID: uid, }) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to update query: %s", err)) diff --git a/api/queryService/query_test.go b/api/queryService/query_test.go index c82cc243..f6b91161 100644 --- a/api/queryService/query_test.go +++ b/api/queryService/query_test.go @@ -167,7 +167,7 @@ func TestGetQueryById(t *testing.T) { pool.ExpectQuery("-- name: GetQuery :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows( pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). - AddRow(database.MustToDBUUID(id), repository.QuerytypeContextFull, int32(1), int32(2), []byte(""), []pgtype.UUID{}), + AddRow(database.MustToDBUUID(id), repository.QuerytypeContextFull, int32(1), int32(2), nil, []pgtype.UUID{}), ) err = cons.GetQueryById(ctx, id.String()) @@ -219,6 +219,11 @@ func TestUpdateQuery(t *testing.T) { AddRow(database.MustToDBUUID(id), repository.QuerytypeContextFull, int32(1), int32(2), []byte(""), []pgtype.UUID{}), ) + pool.ExpectBeginTx(pgx.TxOptions{}) + pool.ExpectExec("name: UpdateQuery :exec").WithArgs(int32(1), int32(2), database.MustToDBUUID(id)). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectCommit() + err = cons.UpdateQuery(ctx, id.String()) assert.Nil(t, err) assert.Equal(t, http.StatusOK, rec.Code) diff --git a/database/migrations/20241219151404_create_activequeryrequirements_view.up.sql b/database/migrations/20241219151404_create_activequeryrequirements_view.up.sql index 3a8162b9..25012662 100644 --- a/database/migrations/20241219151404_create_activequeryrequirements_view.up.sql +++ b/database/migrations/20241219151404_create_activequeryrequirements_view.up.sql @@ -2,4 +2,5 @@ CREATE VIEW activeQueryRequirements AS SELECT q.id, q.type, q.activeVersion, rq.requiredQueryId FROM queries q JOIN requiredQueries rq ON q.id = rq.queryId -WHERE q.activeVersion >= rq.addedVersion AND q.activeVersion < COALESCE(rq.removedVersion, q.activeVersion + 1); \ No newline at end of file + AND q.activeVersion >= rq.addedVersion + AND q.activeVersion < COALESCE(rq.removedVersion, q.activeVersion + 1); \ No newline at end of file diff --git a/database/migrations/20241219161635_create_activecollectorqueries_view.up.sql b/database/migrations/20241219161635_create_activecollectorqueries_view.up.sql index 210cf224..11414b44 100644 --- a/database/migrations/20241219161635_create_activecollectorqueries_view.up.sql +++ b/database/migrations/20241219161635_create_activecollectorqueries_view.up.sql @@ -2,4 +2,5 @@ CREATE VIEW activeCollectorQueries AS SELECT c.id as collectorId, c.activeVersion, cq.queryId FROM collectors c JOIN collectorQueries cq ON c.id = cq.collectorId -WHERE c.activeVersion >= cq.addedVersion AND c.activeVersion < COALESCE(cq.removedVersion, c.activeVersion + 1); \ No newline at end of file + AND c.activeVersion >= cq.addedVersion + AND c.activeVersion < COALESCE(cq.removedVersion, c.activeVersion + 1); \ No newline at end of file diff --git a/database/migrations/20250108174833_create_fullactivequery_view.up.sql b/database/migrations/20250108174833_create_fullactivequery_view.up.sql index 00f7f761..18a2f762 100644 --- a/database/migrations/20250108174833_create_fullactivequery_view.up.sql +++ b/database/migrations/20250108174833_create_fullactivequery_view.up.sql @@ -1,11 +1,10 @@ - CREATE VIEW fullActiveQueries AS - SELECT DISTINCT q.id, q.type, q.activeVersion, q.latestVersion, c.config, ARRAY_AGG(DISTINCT r.requiredQueryId)::uuid[] as requiredIds + SELECT DISTINCT q.id, q.type, q.activeVersion, q.latestVersion, coalesce(c.config, null) as config, ARRAY_AGG(DISTINCT r.requiredQueryId)::uuid[] as requiredIds FROM queries AS q LEFT JOIN queryConfigs AS c ON q.id = c.queryId + and c.addedVersion >= q.activeVersion + and COALESCE(c.removedVersion, q.activeVersion - 1) < q.activeVersion LEFT JOIN requiredQueries AS r ON q.id = r.queryId - WHERE (c.id is null or c.addedVersion >= q.activeVersion - and COALESCE(c.removedVersion, q.activeVersion - 1) < q.activeVersion) - and (r.id is null or r.addedVersion >= q.activeVersion - and COALESCE(r.removedVersion, q.activeVersion - 1) < q.activeVersion) - GROUP BY q.id, q.type, q.activeversion, q.latestversion, c.config; \ No newline at end of file + and r.addedVersion >= q.activeVersion + and COALESCE(r.removedVersion, q.activeVersion - 1) < q.activeVersion + GROUP BY q.id, q.type, q.activeversion, q.latestversion, c.config; diff --git a/database/queries/query.sql b/database/queries/query.sql index efbab606..5707441e 100644 --- a/database/queries/query.sql +++ b/database/queries/query.sql @@ -18,8 +18,22 @@ SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActi -- name: CreateQuery :one INSERT INTO queries (type) VALUES ($1) RETURNING id; --- name: CreateRequiredQuery :exec +-- name: UpdateQuery :exec +UPDATE queries SET activeVersion = $1, latestVersion = $2 WHERE id = $3; + +-- name: AddRequiredQuery :exec INSERT INTO requiredQueries (queryId, requiredQueryId, addedVersion) VALUES ($1, $2, $3); --- name: CreateQueryConfig :exec -INSERT INTO queryConfigs (queryId, config, addedVersion) VALUES ($1, $2, $3); \ No newline at end of file +-- name: RemoveRequiredQuery :exec +UPDATE requiredQueries SET removedVersion = $1 WHERE requiredQueryId = $2 and queryId = $3 and removedVersion is null; + +-- name: AddQueryConfig :exec +INSERT INTO queryConfigs (queryId, config, addedVersion) VALUES ($1, $2, $3); + +-- name: RemoveQueryConfig :exec +UPDATE queryConfigs SET removedVersion = $1 WHERE queryId = $2 and removedVersion is null; + +-- name: AllQueriesExist :one +SELECT COUNT(*) = COUNT(DISTINCT id) AS all_exist + FROM unnest($1::uuid[]) AS input_id + LEFT JOIN queries ON input_id = queries.id; \ No newline at end of file diff --git a/devbox.json b/devbox.json index 5906ecc9..f5d130d6 100644 --- a/devbox.json +++ b/devbox.json @@ -21,12 +21,7 @@ "init_hook": [ "export DB_URI=postgres://${DB_USER}:${DB_PASS}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable", "echo 'Welcome to the DoczyAI devbox!'" - ], - "scripts": { - "init": [ - "task deps:download" - ] - } + ] }, "env": { "APP_ENV": "development", diff --git a/internal/database/parseuuid.go b/internal/database/parseuuid.go index 4861ce44..239fa8ec 100644 --- a/internal/database/parseuuid.go +++ b/internal/database/parseuuid.go @@ -8,33 +8,87 @@ import ( ) func MustToDBUUIDArray(ids []uuid.UUID) []pgtype.UUID { - dbIDs := make([]pgtype.UUID, len(ids)) - for index, id := range ids { - dbIDs[index] = MustToDBUUID(id) - } - - return dbIDs -} - -func MustToDBUUID(id uuid.UUID) pgtype.UUID { - var dbID pgtype.UUID - err := dbID.Scan(id.String()) + dbid, err := ToDBUUIDArray(ids) if err != nil { log.Panic(err) } - return dbID + return dbid } -func MustToUUID(id pgtype.UUID) uuid.UUID { - return uuid.Must(uuid.FromBytes(id.Bytes[:])) +func ToDBUUIDArray(ids []uuid.UUID) ([]pgtype.UUID, error) { + dbIDs := []pgtype.UUID{} + for _, id := range ids { + uid, err := ToDBUUID(id) + if err != nil { + return nil, err + } + + if uid.Valid { + dbIDs = append(dbIDs, uid) + } + } + + return dbIDs, nil } -func MustToUUIDArray(dbIDs []pgtype.UUID) []uuid.UUID { - ids := make([]uuid.UUID, len(dbIDs)) - for index, id := range dbIDs { - ids[index] = MustToUUID(id) +func MustToDBUUID(id uuid.UUID) pgtype.UUID { + dbid, err := ToDBUUID(id) + if err != nil { + log.Panic(err) + } + + return dbid +} + +func ToDBUUID(id uuid.UUID) (pgtype.UUID, error) { + var dbID pgtype.UUID + err := dbID.Scan(id.String()) + if err != nil { + return dbID, err + } + + if id == uuid.Nil { + dbID.Valid = false + } + + return dbID, nil +} + +func MustToUUID(dbid pgtype.UUID) uuid.UUID { + id, err := ToUUID(dbid) + if err != nil { + log.Panic(err) + } + + return id +} + +func ToUUID(id pgtype.UUID) (uuid.UUID, error) { + return uuid.FromBytes(id.Bytes[:]) +} + +func MustToUUIDArray(dbids []pgtype.UUID) []uuid.UUID { + ids, err := ToUUIDArray(dbids) + if err != nil { + log.Panic(err) } return ids } + +func ToUUIDArray(dbIDs []pgtype.UUID) ([]uuid.UUID, error) { + ids := []uuid.UUID{} + for _, id := range dbIDs { + uid, err := ToUUID(id) + if err != nil { + return nil, err + } + + if uid != uuid.Nil { + ids = append(ids, uid) + } + } + + return ids, nil +} diff --git a/internal/database/parseuuid_test.go b/internal/database/parseuuid_test.go index ee1ac14e..4ad97176 100644 --- a/internal/database/parseuuid_test.go +++ b/internal/database/parseuuid_test.go @@ -9,41 +9,53 @@ import ( "github.com/stretchr/testify/assert" ) -func TestMustToDBUUID(t *testing.T) { +func TestToDBUUID(t *testing.T) { id := uuid.New() - dbID := database.MustToDBUUID(id) + dbID, err := database.ToDBUUID(id) + assert.Nil(t, err) - assert.Equal(t, true, dbID.Valid) + assert.True(t, dbID.Valid) assert.Equal(t, id.String(), uuid.UUID(dbID.Bytes).String()) } -func TestMustToDBUUIDArray(t *testing.T) { - ids := []uuid.UUID{uuid.New(), uuid.New()} +func TestToDBUUIDNil(t *testing.T) { + id := uuid.Nil - dbIDs := database.MustToDBUUIDArray(ids) + dbID, err := database.ToDBUUID(id) + assert.Nil(t, err) - assert.Len(t, dbIDs, len(ids)) - for index, id := range dbIDs { - assert.Equal(t, database.MustToDBUUID(ids[index]), id) - } + assert.False(t, dbID.Valid) + assert.Equal(t, id.String(), uuid.UUID(dbID.Bytes).String()) } -func TestMustToUUID(t *testing.T) { - dbID := database.MustToDBUUID(uuid.New()) +func TestToDBUUIDArray(t *testing.T) { + ids := []uuid.UUID{uuid.Nil, uuid.New()} - id := database.MustToUUID(dbID) + dbIDs, err := database.ToDBUUIDArray(ids) + assert.Nil(t, err) + + assert.Len(t, dbIDs, 1) + assert.ElementsMatch(t, []pgtype.UUID{database.MustToDBUUID(ids[1])}, dbIDs) +} + +func TestToUUID(t *testing.T) { + dbID, err := database.ToDBUUID(uuid.New()) + assert.Nil(t, err) + + id, err := database.ToUUID(dbID) + assert.Nil(t, err) assert.Equal(t, id.String(), uuid.UUID(dbID.Bytes).String()) } -func TestMustToUUIDArray(t *testing.T) { - dbIDs := []pgtype.UUID{database.MustToDBUUID(uuid.New()), database.MustToDBUUID(uuid.New())} +func TestToUUIDArray(t *testing.T) { + ogIDs := []uuid.UUID{uuid.Nil, uuid.New()} + dbIDs := database.MustToDBUUIDArray(ogIDs) - ids := database.MustToUUIDArray(dbIDs) + ids, err := database.ToUUIDArray(dbIDs) + assert.Nil(t, err) - assert.Len(t, ids, len(dbIDs)) - for index, id := range dbIDs { - assert.Equal(t, database.MustToDBUUID(ids[index]), id) - } + assert.Len(t, ids, 1) + assert.ElementsMatch(t, []uuid.UUID{ogIDs[1]}, ids) } diff --git a/internal/database/repository/query.sql.go b/internal/database/repository/query.sql.go index ab2afdb5..4b84f4df 100644 --- a/internal/database/repository/query.sql.go +++ b/internal/database/repository/query.sql.go @@ -11,6 +11,60 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const addQueryConfig = `-- name: AddQueryConfig :exec +INSERT INTO queryConfigs (queryId, config, addedVersion) VALUES ($1, $2, $3) +` + +type AddQueryConfigParams struct { + Queryid pgtype.UUID `db:"queryid"` + Config []byte `db:"config"` + Addedversion int32 `db:"addedversion"` +} + +// AddQueryConfig +// +// INSERT INTO queryConfigs (queryId, config, addedVersion) VALUES ($1, $2, $3) +func (q *Queries) AddQueryConfig(ctx context.Context, arg *AddQueryConfigParams) error { + _, err := q.db.Exec(ctx, addQueryConfig, arg.Queryid, arg.Config, arg.Addedversion) + return err +} + +const addRequiredQuery = `-- name: AddRequiredQuery :exec +INSERT INTO requiredQueries (queryId, requiredQueryId, addedVersion) VALUES ($1, $2, $3) +` + +type AddRequiredQueryParams struct { + Queryid pgtype.UUID `db:"queryid"` + Requiredqueryid pgtype.UUID `db:"requiredqueryid"` + Addedversion int32 `db:"addedversion"` +} + +// AddRequiredQuery +// +// INSERT INTO requiredQueries (queryId, requiredQueryId, addedVersion) VALUES ($1, $2, $3) +func (q *Queries) AddRequiredQuery(ctx context.Context, arg *AddRequiredQueryParams) error { + _, err := q.db.Exec(ctx, addRequiredQuery, arg.Queryid, arg.Requiredqueryid, arg.Addedversion) + return err +} + +const allQueriesExist = `-- name: AllQueriesExist :one +SELECT COUNT(*) = COUNT(DISTINCT id) AS all_exist + FROM unnest($1::uuid[]) AS input_id + LEFT JOIN queries ON input_id = queries.id +` + +// AllQueriesExist +// +// SELECT COUNT(*) = COUNT(DISTINCT id) AS all_exist +// FROM unnest($1::uuid[]) AS input_id +// LEFT JOIN queries ON input_id = queries.id +func (q *Queries) AllQueriesExist(ctx context.Context, dollar_1 []pgtype.UUID) (bool, error) { + row := q.db.QueryRow(ctx, allQueriesExist, dollar_1) + var all_exist bool + err := row.Scan(&all_exist) + return all_exist, err +} + const createQuery = `-- name: CreateQuery :one INSERT INTO queries (type) VALUES ($1) RETURNING id ` @@ -25,42 +79,6 @@ func (q *Queries) CreateQuery(ctx context.Context, type_ Querytype) (pgtype.UUID return id, err } -const createQueryConfig = `-- name: CreateQueryConfig :exec -INSERT INTO queryConfigs (queryId, config, addedVersion) VALUES ($1, $2, $3) -` - -type CreateQueryConfigParams struct { - Queryid pgtype.UUID `db:"queryid"` - Config []byte `db:"config"` - Addedversion int32 `db:"addedversion"` -} - -// CreateQueryConfig -// -// INSERT INTO queryConfigs (queryId, config, addedVersion) VALUES ($1, $2, $3) -func (q *Queries) CreateQueryConfig(ctx context.Context, arg *CreateQueryConfigParams) error { - _, err := q.db.Exec(ctx, createQueryConfig, arg.Queryid, arg.Config, arg.Addedversion) - return err -} - -const createRequiredQuery = `-- name: CreateRequiredQuery :exec -INSERT INTO requiredQueries (queryId, requiredQueryId, addedVersion) VALUES ($1, $2, $3) -` - -type CreateRequiredQueryParams struct { - Queryid pgtype.UUID `db:"queryid"` - Requiredqueryid pgtype.UUID `db:"requiredqueryid"` - Addedversion int32 `db:"addedversion"` -} - -// CreateRequiredQuery -// -// INSERT INTO requiredQueries (queryId, requiredQueryId, addedVersion) VALUES ($1, $2, $3) -func (q *Queries) CreateRequiredQuery(ctx context.Context, arg *CreateRequiredQueryParams) error { - _, err := q.db.Exec(ctx, createRequiredQuery, arg.Queryid, arg.Requiredqueryid, arg.Addedversion) - return err -} - const deprecateQuery = `-- name: DeprecateQuery :exec INSERT INTO queryDeprecations (queryId) VALUES ($1) ` @@ -169,3 +187,56 @@ func (q *Queries) ListQueries(ctx context.Context) ([]*Fullactivequery, error) { } return items, nil } + +const removeQueryConfig = `-- name: RemoveQueryConfig :exec +UPDATE queryConfigs SET removedVersion = $1 WHERE queryId = $2 and removedVersion is null +` + +type RemoveQueryConfigParams struct { + Removedversion *int32 `db:"removedversion"` + Queryid pgtype.UUID `db:"queryid"` +} + +// RemoveQueryConfig +// +// UPDATE queryConfigs SET removedVersion = $1 WHERE queryId = $2 and removedVersion is null +func (q *Queries) RemoveQueryConfig(ctx context.Context, arg *RemoveQueryConfigParams) error { + _, err := q.db.Exec(ctx, removeQueryConfig, arg.Removedversion, arg.Queryid) + return err +} + +const removeRequiredQuery = `-- name: RemoveRequiredQuery :exec +UPDATE requiredQueries SET removedVersion = $1 WHERE requiredQueryId = $2 and queryId = $3 and removedVersion is null +` + +type RemoveRequiredQueryParams struct { + Removedversion *int32 `db:"removedversion"` + Requiredqueryid pgtype.UUID `db:"requiredqueryid"` + Queryid pgtype.UUID `db:"queryid"` +} + +// RemoveRequiredQuery +// +// UPDATE requiredQueries SET removedVersion = $1 WHERE requiredQueryId = $2 and queryId = $3 and removedVersion is null +func (q *Queries) RemoveRequiredQuery(ctx context.Context, arg *RemoveRequiredQueryParams) error { + _, err := q.db.Exec(ctx, removeRequiredQuery, arg.Removedversion, arg.Requiredqueryid, arg.Queryid) + return err +} + +const updateQuery = `-- name: UpdateQuery :exec +UPDATE queries SET activeVersion = $1, latestVersion = $2 WHERE id = $3 +` + +type UpdateQueryParams struct { + Activeversion int32 `db:"activeversion"` + Latestversion int32 `db:"latestversion"` + ID pgtype.UUID `db:"id"` +} + +// UpdateQuery +// +// UPDATE queries SET activeVersion = $1, latestVersion = $2 WHERE id = $3 +func (q *Queries) UpdateQuery(ctx context.Context, arg *UpdateQueryParams) error { + _, err := q.db.Exec(ctx, updateQuery, arg.Activeversion, arg.Latestversion, arg.ID) + return err +} diff --git a/internal/database/repository/query_test.go b/internal/database/repository/query_test.go index 369b166c..8662b6e8 100644 --- a/internal/database/repository/query_test.go +++ b/internal/database/repository/query_test.go @@ -9,6 +9,7 @@ import ( "queryorchestration/internal/test" "testing" + "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" "github.com/stretchr/testify/assert" ) @@ -32,22 +33,6 @@ func TestQueries(t *testing.T) { assert.Nil(t, err) assert.True(t, jsonQueryID.Valid) - jsonConfig := []byte("{\"path\": \"example_path\"}") - - err = queries.CreateRequiredQuery(ctx, &repository.CreateRequiredQueryParams{ - Queryid: jsonQueryID, - Requiredqueryid: contextQueryID, - Addedversion: 1, - }) - assert.Nil(t, err) - - err = queries.CreateQueryConfig(ctx, &repository.CreateQueryConfigParams{ - Queryid: jsonQueryID, - Config: jsonConfig, - Addedversion: 1, - }) - assert.Nil(t, err) - jsonQuery, err := queries.GetQuery(ctx, jsonQueryID) assert.Nil(t, err) assert.EqualExportedValues(t, repository.Fullactivequery{ @@ -55,10 +40,41 @@ func TestQueries(t *testing.T) { Type: repository.QuerytypeJsonExtractor, Activeversion: 1, Latestversion: 1, - Config: jsonConfig, - Requiredids: []pgtype.UUID{contextQueryID}, + Config: nil, + Requiredids: []pgtype.UUID{database.MustToDBUUID(uuid.Nil)}, }, *jsonQuery) + err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{ + Latestversion: 2, + Activeversion: 1, + ID: jsonQueryID, + }) + assert.Nil(t, err) + + jsonConfig := []byte("{\"path\": \"example_path\"}") + + err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ + Queryid: jsonQueryID, + Requiredqueryid: contextQueryID, + Addedversion: 1, + }) + assert.Nil(t, err) + + removeV := int32(2) + err = queries.RemoveRequiredQuery(ctx, &repository.RemoveRequiredQueryParams{ + Queryid: jsonQueryID, + Requiredqueryid: contextQueryID, + Removedversion: &removeV, + }) + assert.Nil(t, err) + + err = queries.AddQueryConfig(ctx, &repository.AddQueryConfigParams{ + Queryid: jsonQueryID, + Config: jsonConfig, + Addedversion: 1, + }) + assert.Nil(t, err) + jsonQueryConfig, err := queries.GetQueryConfig(ctx, &repository.GetQueryConfigParams{ Queryid: jsonQueryID, Addedversion: jsonQuery.Activeversion, @@ -66,6 +82,50 @@ func TestQueries(t *testing.T) { assert.Nil(t, err) assert.Equal(t, jsonConfig, jsonQueryConfig.Config) + removeV = 2 + err = queries.RemoveQueryConfig(ctx, &repository.RemoveQueryConfigParams{ + Queryid: jsonQueryID, + Removedversion: &removeV, + }) + assert.Nil(t, err) + + _, err = queries.GetQueryConfig(ctx, &repository.GetQueryConfigParams{ + Queryid: jsonQueryID, + Addedversion: jsonQuery.Activeversion, + }) + assert.EqualError(t, err, "no rows in result set") + + jsonQuery, err = queries.GetQuery(ctx, jsonQueryID) + assert.Nil(t, err) + assert.EqualExportedValues(t, repository.Fullactivequery{ + ID: jsonQueryID, + Type: repository.QuerytypeJsonExtractor, + Activeversion: 1, + Latestversion: 2, + Config: nil, + Requiredids: []pgtype.UUID{database.MustToDBUUID(uuid.Nil)}, + }, *jsonQuery) + + all_exist, err := queries.AllQueriesExist(ctx, []pgtype.UUID{}) + assert.Nil(t, err) + assert.True(t, all_exist) + + all_exist, err = queries.AllQueriesExist(ctx, []pgtype.UUID{database.MustToDBUUID(uuid.New())}) + assert.Nil(t, err) + assert.False(t, all_exist) + + all_exist, err = queries.AllQueriesExist(ctx, []pgtype.UUID{jsonQueryID}) + assert.Nil(t, err) + assert.True(t, all_exist) + + all_exist, err = queries.AllQueriesExist(ctx, []pgtype.UUID{jsonQueryID, contextQueryID}) + assert.Nil(t, err) + assert.True(t, all_exist) + + all_exist, err = queries.AllQueriesExist(ctx, []pgtype.UUID{jsonQueryID, database.MustToDBUUID(uuid.New())}) + assert.Nil(t, err) + assert.False(t, all_exist) + isDeprecated, err := queries.IsQueryDeprecated(ctx, jsonQueryID) assert.Nil(t, err) assert.False(t, isDeprecated) diff --git a/internal/query/create.go b/internal/query/create.go index ede8d12c..a353bb37 100644 --- a/internal/query/create.go +++ b/internal/query/create.go @@ -14,12 +14,7 @@ import ( ) func (s *Service) Create(ctx context.Context, entity *queryprocessor.Create) (uuid.UUID, error) { - validator, err := s.getCreator(entity.Type) - if err != nil { - return uuid.Nil, err - } - - err = validator.Validate(ctx, entity) + err := s.normalizeCreate(ctx, entity) if err != nil { return uuid.Nil, err } @@ -32,6 +27,30 @@ func (s *Service) Create(ctx context.Context, entity *queryprocessor.Create) (uu return id, err } +func (s *Service) normalizeCreate(ctx context.Context, entity *queryprocessor.Create) error { + err := s.normalizeQueryIDs(ctx, entity) + if err != nil { + return err + } + + err = s.normalizeConfig(entity) + if err != nil { + return err + } + + validator, err := s.getCreator(entity.Type) + if err != nil { + return err + } + + err = validator.Validate(ctx, entity) + if err != nil { + return err + } + + return nil +} + func (s *Service) submitCreate(ctx context.Context, entity *queryprocessor.Create) (uuid.UUID, error) { query, err := parseCreateQuery(entity) if err != nil { @@ -53,21 +72,23 @@ func (s *Service) submitCreate(ctx context.Context, entity *queryprocessor.Creat return uuid.Nil, err } - for _, reqQuery := range query.RequiredQueryIDs { - err = qtx.CreateRequiredQuery(ctx, &repository.CreateRequiredQueryParams{ - Queryid: dbID, - Requiredqueryid: reqQuery, - Addedversion: 1, - }) - if err != nil { - return uuid.Nil, err + if query.RequiredQueryIDs != nil { + for _, reqQuery := range *query.RequiredQueryIDs { + err = qtx.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ + Queryid: dbID, + Requiredqueryid: reqQuery, + Addedversion: 1, + }) + if err != nil { + return uuid.Nil, err + } } } - if query.Config != nil && string(query.Config) != "" { - err = qtx.CreateQueryConfig(ctx, &repository.CreateQueryConfigParams{ + if query.Config != nil && string(*query.Config) != "" { + err = qtx.AddQueryConfig(ctx, &repository.AddQueryConfigParams{ Queryid: dbID, - Config: query.Config, + Config: *query.Config, Addedversion: 1, }) if err != nil { @@ -99,8 +120,8 @@ func (s *Service) getCreator(qType queryprocessor.Type) (queryprocessor.Creator, type createQuery struct { Type repository.Querytype - RequiredQueryIDs []pgtype.UUID - Config []byte + RequiredQueryIDs *[]pgtype.UUID + Config *[]byte } func parseCreateQuery(q *queryprocessor.Create) (*createQuery, error) { @@ -109,11 +130,21 @@ func parseCreateQuery(q *queryprocessor.Create) (*createQuery, error) { return nil, err } - reqIDs := database.MustToDBUUIDArray(q.RequiredQueryIDs) + var reqIDs *[]pgtype.UUID + if q.RequiredQueryIDs != nil { + tIDs := database.MustToDBUUIDArray(*q.RequiredQueryIDs) + reqIDs = &tIDs + } + + var cfg *[]byte + if q.Config != nil { + tC := []byte(*q.Config) + cfg = &tC + } return &createQuery{ Type: t, RequiredQueryIDs: reqIDs, - Config: []byte(q.Config), + Config: cfg, }, nil } diff --git a/internal/query/create_test.go b/internal/query/create_test.go index fe64b2a1..6f26ab0b 100644 --- a/internal/query/create_test.go +++ b/internal/query/create_test.go @@ -2,6 +2,7 @@ package query_test import ( "context" + "errors" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/query" @@ -32,10 +33,10 @@ func TestCreate(t *testing.T) { q := query.Query{ ID: uuid.New(), Type: queryprocessor.TypeJsonExtractor, - RequiredQueryIDs: []uuid.UUID{ + RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, - Config: config, + Config: &config, } create := &queryprocessor.Create{ Type: q.Type, @@ -46,16 +47,20 @@ func TestCreate(t *testing.T) { dbType, err := queryprocessor.ToDBQueryType(create.Type) assert.Nil(t, err) + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray(*create.RequiredQueryIDs)).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}).AddRow(true), + ) + pool.ExpectBeginTx(pgx.TxOptions{}) pool.ExpectQuery("name: CreateQuery :one").WithArgs(dbType).WillReturnRows( pgxmock.NewRows([]string{"id"}). AddRow(database.MustToDBUUID(q.ID)), ) - for _, req := range create.RequiredQueryIDs { - pool.ExpectExec("name: CreateRequiredQuery :exec").WithArgs(database.MustToDBUUID(q.ID), database.MustToDBUUID(req), int32(1)). + for _, req := range *create.RequiredQueryIDs { + pool.ExpectExec("name: AddRequiredQuery :exec").WithArgs(database.MustToDBUUID(q.ID), database.MustToDBUUID(req), int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) } - pool.ExpectExec("name: CreateQueryConfig :exec").WithArgs(database.MustToDBUUID(q.ID), []byte(create.Config), int32(1)). + pool.ExpectExec("name: AddQueryConfig :exec").WithArgs(database.MustToDBUUID(q.ID), []byte(*create.Config), int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() @@ -63,3 +68,74 @@ func TestCreate(t *testing.T) { assert.Nil(t, err) assert.Equal(t, q.ID, id) } + +func TestCreateMinimal(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := query.New(db) + + q := query.Query{ + ID: uuid.New(), + Type: queryprocessor.TypeJsonExtractor, + } + create := &queryprocessor.Create{ + Type: q.Type, + } + + dbType, err := queryprocessor.ToDBQueryType(create.Type) + assert.Nil(t, err) + + pool.ExpectBeginTx(pgx.TxOptions{}) + pool.ExpectQuery("name: CreateQuery :one").WithArgs(dbType).WillReturnRows( + pgxmock.NewRows([]string{"id"}). + AddRow(database.MustToDBUUID(q.ID)), + ) + pool.ExpectCommit() + + id, err := svc.Create(ctx, create) + assert.Nil(t, err) + assert.Equal(t, q.ID, id) +} + +func TestCreateRollback(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := query.New(db) + + q := query.Query{ + ID: uuid.New(), + Type: queryprocessor.TypeJsonExtractor, + } + create := &queryprocessor.Create{ + Type: q.Type, + } + + dbType, err := queryprocessor.ToDBQueryType(create.Type) + assert.Nil(t, err) + + pool.ExpectBeginTx(pgx.TxOptions{}) + msg := "database failure" + pool.ExpectQuery("name: CreateQuery :one").WithArgs(dbType).WillReturnError(errors.New(msg)) + pool.ExpectRollback() + + _, err = svc.Create(ctx, create) + assert.EqualError(t, err, msg) +} diff --git a/internal/query/createprivate_test.go b/internal/query/createprivate_test.go index 58f5a1de..4fb166f6 100644 --- a/internal/query/createprivate_test.go +++ b/internal/query/createprivate_test.go @@ -41,30 +41,34 @@ func TestGetCreator(t *testing.T) { } func TestParseCreateQuery(t *testing.T) { + cfg := "{\"key\":\"value\"}" cQuery := &queryprocessor.Create{ Type: queryprocessor.TypeContextFull, - RequiredQueryIDs: []uuid.UUID{ + RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, - Config: "{\"key\":\"value\"}", + Config: &cfg, } resultQuery, err := parseCreateQuery(cQuery) assert.Nil(t, err) + rQIDs := database.MustToDBUUIDArray(*cQuery.RequiredQueryIDs) + qcfg := []byte(*cQuery.Config) assert.EqualExportedValues(t, createQuery{ Type: repository.QuerytypeContextFull, - RequiredQueryIDs: database.MustToDBUUIDArray(cQuery.RequiredQueryIDs), - Config: []byte(cQuery.Config), + RequiredQueryIDs: &rQIDs, + Config: &qcfg, }, *resultQuery) } func TestParseCreateQueryInvalidType(t *testing.T) { + cfg := "{\"key\":\"value\"}" cQuery := &queryprocessor.Create{ Type: queryprocessor.Type(-1), - RequiredQueryIDs: []uuid.UUID{ + RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, - Config: "{\"key\":\"value\"}", + Config: &cfg, } _, err := parseCreateQuery(cQuery) @@ -89,10 +93,10 @@ func TestSubmitCreate(t *testing.T) { q := Query{ ID: uuid.New(), Type: queryprocessor.TypeJsonExtractor, - RequiredQueryIDs: []uuid.UUID{ + RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, - Config: config, + Config: &config, } create := &queryprocessor.Create{ Type: q.Type, @@ -108,11 +112,11 @@ func TestSubmitCreate(t *testing.T) { pgxmock.NewRows([]string{"id"}). AddRow(database.MustToDBUUID(q.ID)), ) - for _, req := range create.RequiredQueryIDs { - pool.ExpectExec("name: CreateRequiredQuery :exec").WithArgs(database.MustToDBUUID(q.ID), database.MustToDBUUID(req), int32(1)). + for _, req := range *create.RequiredQueryIDs { + pool.ExpectExec("name: AddRequiredQuery :exec").WithArgs(database.MustToDBUUID(q.ID), database.MustToDBUUID(req), int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) } - pool.ExpectExec("name: CreateQueryConfig :exec").WithArgs(database.MustToDBUUID(q.ID), []byte(create.Config), int32(1)). + pool.ExpectExec("name: AddQueryConfig :exec").WithArgs(database.MustToDBUUID(q.ID), []byte(*create.Config), int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() @@ -120,3 +124,77 @@ func TestSubmitCreate(t *testing.T) { assert.Nil(t, err) assert.Equal(t, q.ID, id) } + +func TestSubmitCreateNoReqsOrConfig(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := New(db) + + q := Query{ + ID: uuid.New(), + Type: queryprocessor.TypeJsonExtractor, + } + create := &queryprocessor.Create{ + Type: q.Type, + } + + dbType, err := queryprocessor.ToDBQueryType(create.Type) + assert.Nil(t, err) + + pool.ExpectBeginTx(pgx.TxOptions{}) + pool.ExpectQuery("name: CreateQuery :one").WithArgs(dbType).WillReturnRows( + pgxmock.NewRows([]string{"id"}). + AddRow(database.MustToDBUUID(q.ID)), + ) + pool.ExpectCommit() + + id, err := svc.submitCreate(ctx, create) + assert.Nil(t, err) + assert.Equal(t, q.ID, id) +} + +func TestNormalizeCreate(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := New(db) + + cfg := "{}" + create := &queryprocessor.Create{ + Type: queryprocessor.TypeJsonExtractor, + Config: &cfg, + RequiredQueryIDs: &[]uuid.UUID{}, + } + + dbids := database.MustToDBUUIDArray(*create.RequiredQueryIDs) + + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(true), + ) + + err = svc.normalizeCreate(ctx, create) + assert.Nil(t, err) + assert.EqualExportedValues(t, queryprocessor.Create{ + Type: queryprocessor.TypeJsonExtractor, + Config: nil, + RequiredQueryIDs: nil, + }, *create) +} diff --git a/internal/query/document/sync_test.go b/internal/query/document/sync_test.go index 45db40d8..54817645 100644 --- a/internal/query/document/sync_test.go +++ b/internal/query/document/sync_test.go @@ -129,18 +129,22 @@ func TestSyncDBFail(t *testing.T) { pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}). AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion), ) + qV := int32(1) + reqID := database.MustToDBUUID(uuid.New()) + resID := database.MustToDBUUID(uuid.New()) pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion). WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}), + pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}). + AddRow(resID, reqID, qV), ) dbErr = "database failure" - qV := int32(1) pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorId). WillReturnRows( pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}). - AddRow(dbCollectorId, dbQueryID, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, &qV, []pgtype.UUID{}), + AddRow(dbCollectorId, dbQueryID, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, &qV, []pgtype.UUID{reqID}). + AddRow(dbCollectorId, reqID, repository.NullQuerytype{Querytype: repository.QuerytypeContextFull, Valid: true}, &qV, []pgtype.UUID{}), ) - pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{}). + pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{resID}). WillReturnError(errors.New(dbErr)) docSvc = document.New(db) diff --git a/internal/query/get.go b/internal/query/get.go index c4f447d6..6f87ae3f 100644 --- a/internal/query/get.go +++ b/internal/query/get.go @@ -14,8 +14,8 @@ type Query struct { Type queryprocessor.Type ActiveVersion int32 LatestVersion int32 - RequiredQueryIDs []uuid.UUID - Config string + RequiredQueryIDs *[]uuid.UUID + Config *string } func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Query, error) { @@ -28,9 +28,13 @@ func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Query, error) { } func ParseFullActiveQuery(q *repository.Fullactivequery) (*Query, error) { - reqQueryIDs := []uuid.UUID{} - if !(len(q.Requiredids) == 0 || len(q.Requiredids) == 1 && database.MustToUUID(q.Requiredids[0]) == uuid.Nil) { - reqQueryIDs = database.MustToUUIDArray(q.Requiredids) + var reqQueryIDs *[]uuid.UUID + if len(q.Requiredids) > 0 { + rQ, err := database.ToUUIDArray(q.Requiredids) + if err != nil { + return nil, err + } + reqQueryIDs = &rQ } qType, err := queryprocessor.ParseDBType(q.Type) @@ -38,13 +42,19 @@ func ParseFullActiveQuery(q *repository.Fullactivequery) (*Query, error) { return nil, err } + var scfg *string + if q.Config != nil && string(q.Config) != "" { + s := string(q.Config) + scfg = &s + } + return &Query{ ID: database.MustToUUID(q.ID), ActiveVersion: q.Activeversion, LatestVersion: q.Latestversion, Type: qType, RequiredQueryIDs: reqQueryIDs, - Config: string(q.Config), + Config: scfg, }, nil } diff --git a/internal/query/get_test.go b/internal/query/get_test.go index c0de3da6..a68c4f24 100644 --- a/internal/query/get_test.go +++ b/internal/query/get_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) @@ -34,13 +33,13 @@ func TestGet(t *testing.T) { Type: queryprocessor.TypeJsonExtractor, ActiveVersion: int32(1), LatestVersion: int32(1), - RequiredQueryIDs: []uuid.UUID{ + RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, - Config: config, + Config: &config, } - dbReqIDs := database.MustToDBUUIDArray(query.RequiredQueryIDs) + dbReqIDs := database.MustToDBUUIDArray(*query.RequiredQueryIDs) pool.ExpectQuery("name: GetQuery :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows( pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). @@ -59,19 +58,15 @@ func TestFullActiveQueryEmpty(t *testing.T) { Type: repository.QuerytypeContextFull, Activeversion: int32(1), Latestversion: int32(2), - Config: []byte(""), - Requiredids: []pgtype.UUID{}, } out, err := query.ParseFullActiveQuery(dbQuery) assert.Nil(t, err) assert.EqualExportedValues(t, query.Query{ - ID: database.MustToUUID(dbQuery.ID), - Type: queryprocessor.TypeContextFull, - ActiveVersion: int32(1), - LatestVersion: int32(2), - Config: "", - RequiredQueryIDs: []uuid.UUID{}, + ID: database.MustToUUID(dbQuery.ID), + Type: queryprocessor.TypeContextFull, + ActiveVersion: int32(1), + LatestVersion: int32(2), }, *out) } @@ -81,19 +76,15 @@ func TestFullActiveQueryWithNullUUID(t *testing.T) { Type: repository.QuerytypeContextFull, Activeversion: int32(1), Latestversion: int32(2), - Config: []byte(""), - Requiredids: []pgtype.UUID{database.MustToDBUUID(uuid.Nil)}, } out, err := query.ParseFullActiveQuery(dbQuery) assert.Nil(t, err) assert.EqualExportedValues(t, query.Query{ - ID: database.MustToUUID(dbQuery.ID), - Type: queryprocessor.TypeContextFull, - ActiveVersion: int32(1), - LatestVersion: int32(2), - Config: "", - RequiredQueryIDs: []uuid.UUID{}, + ID: database.MustToUUID(dbQuery.ID), + Type: queryprocessor.TypeContextFull, + ActiveVersion: int32(1), + LatestVersion: int32(2), }, *out) } @@ -104,8 +95,6 @@ func TestFullActiveQueryArray(t *testing.T) { Type: repository.QuerytypeContextFull, Activeversion: int32(1), Latestversion: int32(2), - Config: []byte(""), - Requiredids: []pgtype.UUID{}, }, } @@ -113,12 +102,10 @@ func TestFullActiveQueryArray(t *testing.T) { assert.Nil(t, err) assert.EqualExportedValues(t, []*query.Query{ { - ID: database.MustToUUID(dbQueries[0].ID), - Type: queryprocessor.TypeContextFull, - ActiveVersion: int32(1), - LatestVersion: int32(2), - Config: "", - RequiredQueryIDs: []uuid.UUID{}, + ID: database.MustToUUID(dbQueries[0].ID), + Type: queryprocessor.TypeContextFull, + ActiveVersion: int32(1), + LatestVersion: int32(2), }, }, out) } diff --git a/internal/query/list_test.go b/internal/query/list_test.go index bd159512..9c0c86b9 100644 --- a/internal/query/list_test.go +++ b/internal/query/list_test.go @@ -33,13 +33,13 @@ func TestList(t *testing.T) { Type: queryprocessor.TypeJsonExtractor, ActiveVersion: int32(1), LatestVersion: int32(1), - RequiredQueryIDs: []uuid.UUID{ + RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, - Config: config, + Config: &config, } - dbReqIDs := database.MustToDBUUIDArray(q.RequiredQueryIDs) + dbReqIDs := database.MustToDBUUIDArray(*q.RequiredQueryIDs) filters := query.ListFilters{} @@ -74,13 +74,13 @@ func TestListFilterType(t *testing.T) { Type: queryprocessor.TypeJsonExtractor, ActiveVersion: int32(1), LatestVersion: int32(1), - RequiredQueryIDs: []uuid.UUID{ + RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, - Config: config, + Config: &config, } - dbReqIDs := database.MustToDBUUIDArray(q.RequiredQueryIDs) + dbReqIDs := database.MustToDBUUIDArray(*q.RequiredQueryIDs) filters := query.ListFilters{} diff --git a/internal/query/normalize.go b/internal/query/normalize.go new file mode 100644 index 00000000..fe5e3d56 --- /dev/null +++ b/internal/query/normalize.go @@ -0,0 +1,105 @@ +package query + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "queryorchestration/internal/database" + queryprocessor "queryorchestration/internal/query/processor" + "strings" + + "github.com/google/uuid" +) + +type Config interface { + GetConfig() *string + SetConfig(*string) +} + +func (s *Service) normalizeConfig(config Config) error { + if config == nil || config.GetConfig() == nil { + return nil + } + + trim := strings.TrimSpace(*config.GetConfig()) + + if trim == "" { + config.SetConfig(nil) + return nil + } + + if !json.Valid([]byte(trim)) { + return errors.New("invalid config JSON") + } + + var data map[string]interface{} + if err := json.Unmarshal([]byte(trim), &data); err != nil { + return fmt.Errorf("error unmarshalling JSON: %s", err) + } + + prettyJSON, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("error marshalling JSON: %s", err) + } + + strJSON := string(prettyJSON) + + if strJSON == "{}" { + config.SetConfig(nil) + } else { + config.SetConfig(&strJSON) + } + + return nil +} + +type RequiredQueryIDs interface { + GetRequiredQueryIDs() *[]uuid.UUID + SetRequiredQueryIDs(*[]uuid.UUID) +} + +func (s *Service) normalizeQueryIDs(ctx context.Context, ids RequiredQueryIDs) error { + if ids == nil || ids.GetRequiredQueryIDs() == nil { + return nil + } + + ide := ids.GetRequiredQueryIDs() + + if len(*ide) == 0 { + ids.SetRequiredQueryIDs(nil) + return nil + } + + dbids := database.MustToDBUUIDArray(*ide) + + exist, err := s.db.Queries.AllQueriesExist(ctx, dbids) + if err != nil { + return err + } else if !exist { + return errors.New("not all required ids are present") + } + + return nil +} + +func (s *Service) normalizeActiveVersion(current *Query, entity *queryprocessor.Update) error { + if current == nil { + return errors.New("current query required") + } + + if entity == nil || entity.ActiveVersion == nil { + return nil + } + + if entity.ActiveVersion == ¤t.ActiveVersion { + entity.ActiveVersion = nil + return nil + } + + if *entity.ActiveVersion < 1 || *entity.ActiveVersion > current.LatestVersion+1 { + return fmt.Errorf("active version must be in the range: 1 <= activeVersion <= %d", current.LatestVersion+1) + } + + return nil +} diff --git a/internal/query/normalize_test.go b/internal/query/normalize_test.go new file mode 100644 index 00000000..8ab3d517 --- /dev/null +++ b/internal/query/normalize_test.go @@ -0,0 +1,170 @@ +package query + +import ( + "context" + "errors" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + queryprocessor "queryorchestration/internal/query/processor" + "testing" + + "github.com/google/uuid" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" +) + +func TestNormalizeConfig(t *testing.T) { + s := Service{} + + err := s.normalizeConfig(nil) + assert.Nil(t, err) + + entity := queryprocessor.Create{} + + entity.Config = nil + err = s.normalizeConfig(&entity) + assert.Nil(t, err) + assert.Nil(t, entity.Config) + + cfg := "" + entity.Config = &cfg + err = s.normalizeConfig(&entity) + assert.Nil(t, err) + assert.Nil(t, entity.Config) + + cfg = " " + entity.Config = &cfg + err = s.normalizeConfig(&entity) + assert.Nil(t, err) + assert.Nil(t, entity.Config) + + cfg = "{}" + entity.Config = &cfg + err = s.normalizeConfig(&entity) + assert.Nil(t, err) + assert.Nil(t, entity.Config) + + cfg = "{\"hello\":\"bye\"}" + entity.Config = &cfg + err = s.normalizeConfig(&entity) + assert.Nil(t, err) + assert.Equal(t, "{\"hello\":\"bye\"}", *(entity.Config)) + + cfg = " { \"hello\" : \"bye\" } " + entity.Config = &cfg + err = s.normalizeConfig(&entity) + assert.Nil(t, err) + assert.Equal(t, "{\"hello\":\"bye\"}", *(entity.Config)) + + cfg = "{'hello':'bye'}" + entity.Config = &cfg + err = s.normalizeConfig(&entity) + assert.Error(t, err) + + cfg = "{\"hello\":\"}" + entity.Config = &cfg + err = s.normalizeConfig(&entity) + assert.Error(t, err) +} + +func TestNormalizeQueryIDs(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + s := Service{db: db} + + err = s.normalizeQueryIDs(ctx, nil) + assert.Nil(t, err) + + entity := queryprocessor.Create{} + + entity.RequiredQueryIDs = nil + err = s.normalizeQueryIDs(ctx, &entity) + assert.Nil(t, err) + assert.Nil(t, entity.RequiredQueryIDs) + + entity.RequiredQueryIDs = &[]uuid.UUID{} + err = s.normalizeQueryIDs(ctx, &entity) + assert.Nil(t, err) + assert.Nil(t, entity.RequiredQueryIDs) + + ids := []uuid.UUID{uuid.New()} + entity.RequiredQueryIDs = &ids + dbids := database.MustToDBUUIDArray(*entity.RequiredQueryIDs) + + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(true), + ) + + err = s.normalizeQueryIDs(ctx, &entity) + assert.Nil(t, err) + assert.ElementsMatch(t, ids, *entity.RequiredQueryIDs) + + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(false), + ) + + err = s.normalizeQueryIDs(ctx, &entity) + assert.Error(t, err) + assert.ElementsMatch(t, ids, *entity.RequiredQueryIDs) + + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids). + WillReturnError(errors.New("database failure")) + + err = s.normalizeQueryIDs(ctx, &entity) + assert.Error(t, err) + assert.ElementsMatch(t, ids, *entity.RequiredQueryIDs) +} + +func TestNormalizeActiveVersion(t *testing.T) { + s := Service{} + + err := s.normalizeActiveVersion(nil, nil) + assert.Error(t, err) + + current := Query{ + ActiveVersion: 2, + LatestVersion: 4, + } + entity := queryprocessor.Update{} + + err = s.normalizeActiveVersion(nil, &entity) + assert.Error(t, err) + + err = s.normalizeActiveVersion(¤t, nil) + assert.Nil(t, err) + + err = s.normalizeActiveVersion(¤t, &entity) + assert.Nil(t, err) + + entity.ActiveVersion = ¤t.ActiveVersion + err = s.normalizeActiveVersion(¤t, &entity) + assert.Nil(t, err) + assert.Nil(t, entity.ActiveVersion) + + aV := current.ActiveVersion + 1 + entity.ActiveVersion = &aV + err = s.normalizeActiveVersion(¤t, &entity) + assert.Nil(t, err) + assert.Equal(t, aV, *entity.ActiveVersion) + + aV = current.LatestVersion + 2 + entity.ActiveVersion = &aV + err = s.normalizeActiveVersion(¤t, &entity) + assert.Error(t, err) + + aV = 0 + entity.ActiveVersion = &aV + err = s.normalizeActiveVersion(¤t, &entity) + assert.Error(t, err) +} diff --git a/internal/query/parse_test.go b/internal/query/parse_test.go index e5f746e4..7f76faeb 100644 --- a/internal/query/parse_test.go +++ b/internal/query/parse_test.go @@ -13,15 +13,16 @@ import ( ) func TestParseQuery(t *testing.T) { + cfg := "example" q := &query.Query{ ID: uuid.New(), Type: queryprocessor.TypeJsonExtractor, ActiveVersion: int32(1), LatestVersion: int32(2), - RequiredQueryIDs: []uuid.UUID{ + RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, - Config: "example", + Config: &cfg, } out := query.ParseQuery(q) @@ -48,14 +49,15 @@ func TestParseFullActiveQuery(t *testing.T) { out, err := query.ParseFullActiveQuery(q) assert.Nil(t, err) + bcfg := string(q.Config) assert.EqualExportedValues(t, query.Query{ ID: database.MustToUUID(q.ID), Type: queryprocessor.TypeContextFull, ActiveVersion: q.Activeversion, LatestVersion: q.Latestversion, - RequiredQueryIDs: []uuid.UUID{ + RequiredQueryIDs: &[]uuid.UUID{ database.MustToUUID(q.Requiredids[0]), }, - Config: string(q.Config), + Config: &bcfg, }, *out) } diff --git a/internal/query/processor/parse.go b/internal/query/processor/parse.go index 45f7184a..07fdb995 100644 --- a/internal/query/processor/parse.go +++ b/internal/query/processor/parse.go @@ -4,6 +4,8 @@ import ( "fmt" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + + "github.com/google/uuid" ) func ParseDBNullType(qType repository.NullQuerytype) (Type, error) { @@ -64,7 +66,11 @@ func ToDBNullQueryType(t Type) (repository.NullQuerytype, error) { } func ParseDBCollectorQuery(q *repository.GetCollectorQueriesRow) (*Query, error) { - reqQueryIDs := database.MustToUUIDArray(q.Requiredids) + var reqQueryIDs *[]uuid.UUID + if len(q.Requiredids) > 0 { + ids := database.MustToUUIDArray(q.Requiredids) + reqQueryIDs = &ids + } qType, err := ParseDBNullType(q.Type) if err != nil { diff --git a/internal/query/processor/parse_test.go b/internal/query/processor/parse_test.go index b0a9c2ca..c019d846 100644 --- a/internal/query/processor/parse_test.go +++ b/internal/query/processor/parse_test.go @@ -22,7 +22,7 @@ func TestParseDBCollectorQuery(t *testing.T) { value, err := queryprocessor.ParseDBCollectorQuery(&dbResult) assert.Nil(t, err) assert.Equal(t, uuid.Nil, value.ID) - assert.Equal(t, []uuid.UUID{}, value.RequiredQueryIDs) + assert.Nil(t, value.RequiredQueryIDs) assert.Equal(t, int32(0), value.Version) assert.Equal(t, queryprocessor.Type(queryprocessor.TypeJsonExtractor), value.Type) diff --git a/internal/query/processor/service.go b/internal/query/processor/service.go index 77109403..599a3bdc 100644 --- a/internal/query/processor/service.go +++ b/internal/query/processor/service.go @@ -16,22 +16,55 @@ const ( type Create struct { Type Type - RequiredQueryIDs []uuid.UUID - Config string + RequiredQueryIDs *[]uuid.UUID + Config *string +} + +func (s *Create) GetConfig() *string { + return s.Config +} + +func (s *Create) SetConfig(cfg *string) { + s.Config = cfg +} + +func (s *Create) GetRequiredQueryIDs() *[]uuid.UUID { + return s.RequiredQueryIDs +} + +func (s *Create) SetRequiredQueryIDs(ids *[]uuid.UUID) { + s.RequiredQueryIDs = ids } type Update struct { ID uuid.UUID - RequiredQueryIDs []uuid.UUID - Config string + ActiveVersion *int32 + RequiredQueryIDs *[]uuid.UUID + Config *string +} + +func (s *Update) GetConfig() *string { + return s.Config +} + +func (s *Update) SetConfig(cfg *string) { + s.Config = cfg +} + +func (s *Update) GetRequiredQueryIDs() *[]uuid.UUID { + return s.RequiredQueryIDs +} + +func (s *Update) SetRequiredQueryIDs(ids *[]uuid.UUID) { + s.RequiredQueryIDs = ids } type Query struct { ID uuid.UUID Type Type Version int32 - RequiredQueryIDs []uuid.UUID - Config string + RequiredQueryIDs *[]uuid.UUID + Config *string } type Creator interface { @@ -43,5 +76,5 @@ type Updator interface { } type Processor interface { - Process(ctx context.Context, query *Query, values []result.Value) (string, error) + Process(ctx context.Context, query *Query, values *[]result.Value) (string, error) } diff --git a/internal/query/processor/service_test.go b/internal/query/processor/service_test.go new file mode 100644 index 00000000..d4f2c73b --- /dev/null +++ b/internal/query/processor/service_test.go @@ -0,0 +1,93 @@ +package queryprocessor_test + +import ( + queryprocessor "queryorchestration/internal/query/processor" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" +) + +func TestCreateGetConfig(t *testing.T) { + entity := queryprocessor.Create{} + + assert.Nil(t, entity.GetConfig()) + + cfg := "example_config" + entity.Config = &cfg + assert.NotNil(t, entity.GetConfig()) + assert.Equal(t, cfg, *entity.GetConfig()) +} + +func TestCreateSetConfig(t *testing.T) { + entity := queryprocessor.Create{} + + assert.Nil(t, entity.Config) + + cfg := "example_config" + entity.SetConfig(&cfg) + assert.Equal(t, cfg, *entity.Config) +} + +func TestUpdateGetConfig(t *testing.T) { + entity := queryprocessor.Update{} + + assert.Nil(t, entity.GetConfig()) + + cfg := "example_config" + entity.Config = &cfg + assert.NotNil(t, entity.GetConfig()) + assert.Equal(t, cfg, *entity.GetConfig()) +} + +func TestUpdateSetConfig(t *testing.T) { + entity := queryprocessor.Update{} + + assert.Nil(t, entity.Config) + + cfg := "example_config" + entity.SetConfig(&cfg) + assert.Equal(t, cfg, *entity.Config) +} + +func TestCreateGetRequiredQueryIDs(t *testing.T) { + entity := queryprocessor.Create{} + + assert.Nil(t, entity.GetRequiredQueryIDs()) + + ids := []uuid.UUID{uuid.New()} + entity.RequiredQueryIDs = &ids + assert.NotNil(t, entity.GetRequiredQueryIDs()) + assert.Equal(t, ids, *entity.GetRequiredQueryIDs()) +} + +func TestCreateSetRequiredQueryIDs(t *testing.T) { + entity := queryprocessor.Create{} + + assert.Nil(t, entity.RequiredQueryIDs) + + ids := []uuid.UUID{uuid.New()} + entity.SetRequiredQueryIDs(&ids) + assert.Equal(t, ids, *entity.RequiredQueryIDs) +} + +func TestUpdateGetRequiredQueryIDs(t *testing.T) { + entity := queryprocessor.Update{} + + assert.Nil(t, entity.GetRequiredQueryIDs()) + + ids := []uuid.UUID{uuid.New()} + entity.RequiredQueryIDs = &ids + assert.NotNil(t, entity.GetRequiredQueryIDs()) + assert.Equal(t, ids, *entity.GetRequiredQueryIDs()) +} + +func TestUpdateSetRequiredQueryIDs(t *testing.T) { + entity := queryprocessor.Update{} + + assert.Nil(t, entity.RequiredQueryIDs) + + ids := []uuid.UUID{uuid.New()} + entity.SetRequiredQueryIDs(&ids) + assert.Equal(t, ids, *entity.RequiredQueryIDs) +} diff --git a/internal/query/queue/create.go b/internal/query/queue/create.go index 4a52b653..ada56a27 100644 --- a/internal/query/queue/create.go +++ b/internal/query/queue/create.go @@ -64,20 +64,24 @@ func (q *Queue) Add(qu *queryprocessor.Query) { if entry.ID == qu.ID { return } - for _, id := range qu.RequiredQueryIDs { - if entry.ID == id { - requiredIndex = index - break + if qu.RequiredQueryIDs != nil { + for _, id := range *qu.RequiredQueryIDs { + if entry.ID == id { + requiredIndex = index + break + } } } } } for _, entry := range q.collectorQueries { - for _, id := range entry.RequiredQueryIDs { - if qu.ID == id { - dependentQueries = append(dependentQueries, entry) - break + if entry.RequiredQueryIDs != nil { + for _, id := range *entry.RequiredQueryIDs { + if qu.ID == id { + dependentQueries = append(dependentQueries, entry) + break + } } } } diff --git a/internal/query/queue/create_test.go b/internal/query/queue/create_test.go index 7aa73a7b..004ebd1c 100644 --- a/internal/query/queue/create_test.go +++ b/internal/query/queue/create_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) @@ -60,13 +61,16 @@ func TestGetCollectorQueries(t *testing.T) { dbCollectorID := database.MustToDBUUID(svc.collector.ID) collectorQueries := []*queryprocessor.Query{ - {ID: uuid.New(), Type: queryprocessor.TypeContextFull, RequiredQueryIDs: []uuid.UUID{}, Version: int32(1)}, + {ID: uuid.New(), Type: queryprocessor.TypeContextFull, Version: int32(1)}, } rows := pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}) for _, q := range collectorQueries { dbID := database.MustToDBUUID(q.ID) - dbReqIDs := database.MustToDBUUIDArray(q.RequiredQueryIDs) + dbReqIDs := []pgtype.UUID{} + if q.RequiredQueryIDs != nil { + dbReqIDs = database.MustToDBUUIDArray(*q.RequiredQueryIDs) + } ty, err := queryprocessor.ToDBNullQueryType(q.Type) assert.Nil(t, err) rows = rows. diff --git a/internal/query/queue/execute.go b/internal/query/queue/execute.go index a26b5999..6f09f591 100644 --- a/internal/query/queue/execute.go +++ b/internal/query/queue/execute.go @@ -6,6 +6,7 @@ import ( queryprocessor "queryorchestration/internal/query/processor" "queryorchestration/internal/query/result" + "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" ) @@ -27,43 +28,57 @@ func (q *Queue) Execute(ctx context.Context) error { } func (q *Queue) executeQuery(ctx context.Context, qu *queryprocessor.Query) error { - resultIDs := make([]pgtype.UUID, len(qu.RequiredQueryIDs)) - for index, id := range qu.RequiredQueryIDs { - var queryVersion int32 - for _, entry := range q.collectorQueries { - if entry.ID == id { - queryVersion = entry.Version - break - } - } - - for _, entry := range q.results { - if entry.QueryID == id && entry.QueryVersion == queryVersion { - resultIDs[index] = database.MustToDBUUID(entry.ID) - break - } - } - } - - values, err := q.db.Queries.ListResultValuesByID(ctx, resultIDs) + values, err := q.getRequiredResults(ctx, qu.RequiredQueryIDs) if err != nil { return err } - cleanValues := make([]result.Value, len(values)) - for index, r := range values { - cleanValue, err := q.getResultValue(r) - if err != nil { - return err - } - - cleanValues[index] = cleanValue - } - - err = q.setResult(ctx, qu, cleanValues) + err = q.setResult(ctx, qu, values) if err != nil { return err } return nil } + +func (q *Queue) getRequiredResults(ctx context.Context, requiredQueryIDs *[]uuid.UUID) (*[]result.Value, error) { + var values *[]result.Value + if requiredQueryIDs != nil { + resultIDs := make([]pgtype.UUID, len(*requiredQueryIDs)) + for index, id := range *requiredQueryIDs { + var queryVersion int32 + for _, entry := range q.collectorQueries { + if entry.ID == id { + queryVersion = entry.Version + break + } + } + + for _, entry := range q.results { + if entry.QueryID == id && entry.QueryVersion == queryVersion { + resultIDs[index] = database.MustToDBUUID(entry.ID) + break + } + } + } + + resultValues, err := q.db.Queries.ListResultValuesByID(ctx, resultIDs) + if err != nil { + return nil, err + } + + rValues := make([]result.Value, len(resultValues)) + for index, r := range resultValues { + cleanValue, err := q.getResultValue(r) + if err != nil { + return nil, err + } + + rValues[index] = cleanValue + } + + values = &rValues + } + + return values, nil +} diff --git a/internal/query/queue/execute_test.go b/internal/query/queue/execute_test.go index 78914e82..f5e23f37 100644 --- a/internal/query/queue/execute_test.go +++ b/internal/query/queue/execute_test.go @@ -58,19 +58,22 @@ func TestExecute(t *testing.T) { contextID := uuid.New() contextVersion := int32(1) collectorQueries := []queryprocessor.Query{ - {ID: contextID, Type: queryprocessor.TypeContextFull, RequiredQueryIDs: []uuid.UUID{}, Version: contextVersion}, - {ID: queryOneID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{contextID}, Version: queryOneVersion}, - {ID: queryTwoID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{queryOneID}, Version: queryTwoVersion}, - {ID: queryThreeID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{queryOneID}, Version: queryThreeVersion}, - {ID: queryFourID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{contextID}, Version: queryFourVersion}, - {ID: queryFiveID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{querySixID}, Version: queryFiveVersion}, - {ID: querySixID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{contextID}, Version: querySixVersion}, + {ID: contextID, Type: queryprocessor.TypeContextFull, Version: contextVersion}, + {ID: queryOneID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: queryOneVersion}, + {ID: queryTwoID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryTwoVersion}, + {ID: queryThreeID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryThreeVersion}, + {ID: queryFourID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: queryFourVersion}, + {ID: queryFiveID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{querySixID}, Version: queryFiveVersion}, + {ID: querySixID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: querySixVersion}, } rows := pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}) for _, q := range collectorQueries { dbID := database.MustToDBUUID(q.ID) - dbReqIDs := database.MustToDBUUIDArray(q.RequiredQueryIDs) + dbReqIDs := []pgtype.UUID{} + if q.RequiredQueryIDs != nil { + dbReqIDs = database.MustToDBUUIDArray(*q.RequiredQueryIDs) + } ty, err := queryprocessor.ToDBNullQueryType(q.Type) assert.Nil(t, err) rows = rows. @@ -88,11 +91,11 @@ func TestExecute(t *testing.T) { } expectedQueries := []*queryprocessor.Query{ - {ID: querySixID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{contextID}, Version: querySixVersion}, - {ID: queryFiveID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{querySixID}, Version: queryFiveVersion}, - {ID: queryOneID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{contextID}, Version: queryOneVersion}, - {ID: queryThreeID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{queryOneID}, Version: queryThreeVersion}, - {ID: queryTwoID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{queryOneID}, Version: queryTwoVersion}, + {ID: querySixID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: querySixVersion}, + {ID: queryFiveID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{querySixID}, Version: queryFiveVersion}, + {ID: queryOneID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: queryOneVersion}, + {ID: queryThreeID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryThreeVersion}, + {ID: queryTwoID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryTwoVersion}, } docID := uuid.New() diff --git a/internal/query/queue/executeprivate_test.go b/internal/query/queue/executeprivate_test.go index 770ec982..f2a291ce 100644 --- a/internal/query/queue/executeprivate_test.go +++ b/internal/query/queue/executeprivate_test.go @@ -27,8 +27,8 @@ func TestExecute(t *testing.T) { } expectedQueries := []*queryprocessor.Query{ - {ID: uuid.New(), Type: queryprocessor.TypeContextFull, RequiredQueryIDs: []uuid.UUID{}, Version: int32(1)}, - {ID: uuid.New(), Type: queryprocessor.TypeContextFull, RequiredQueryIDs: []uuid.UUID{}, Version: int32(2)}, + {ID: uuid.New(), Type: queryprocessor.TypeContextFull, Version: int32(1)}, + {ID: uuid.New(), Type: queryprocessor.TypeContextFull, Version: int32(2)}, } q := &Queue{ @@ -39,17 +39,11 @@ func TestExecute(t *testing.T) { textVersion: int32(2), } - pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{}).WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "value"}), - ) pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(expectedQueries[0].ID), database.MustToDBUUID(q.documentId), pgxmock.AnyArg(), q.cleanVersion, q.textVersion, expectedQueries[0].Version). WillReturnRows( pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), ) - pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{}).WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "value"}), - ) pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(expectedQueries[1].ID), database.MustToDBUUID(q.documentId), pgxmock.AnyArg(), q.cleanVersion, q.textVersion, expectedQueries[1].Version). WillReturnRows( pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), @@ -72,7 +66,7 @@ func TestExecuteQuery(t *testing.T) { Pool: pool, } - qu := &queryprocessor.Query{ID: uuid.New(), Type: queryprocessor.TypeContextFull, RequiredQueryIDs: []uuid.UUID{}, Version: int32(1)} + qu := &queryprocessor.Query{ID: uuid.New(), Type: queryprocessor.TypeContextFull, Version: int32(1)} q := &Queue{ db: db, @@ -81,9 +75,6 @@ func TestExecuteQuery(t *testing.T) { textVersion: int32(2), } - pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{}).WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "value"}), - ) pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(qu.ID), database.MustToDBUUID(q.documentId), pgxmock.AnyArg(), q.cleanVersion, q.textVersion, qu.Version). WillReturnRows( pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), diff --git a/internal/query/queue/result.go b/internal/query/queue/result.go index a9ee716a..0906f4f2 100644 --- a/internal/query/queue/result.go +++ b/internal/query/queue/result.go @@ -11,7 +11,7 @@ import ( jsonextractor "queryorchestration/internal/query/types/jsonExtractor" ) -func (q *Queue) setResult(ctx context.Context, qu *queryprocessor.Query, resultValues []result.Value) error { +func (q *Queue) setResult(ctx context.Context, qu *queryprocessor.Query, resultValues *[]result.Value) error { processor, err := q.getProcessor(qu.Type) if err != nil { return err diff --git a/internal/query/queue/resultprivate_test.go b/internal/query/queue/resultprivate_test.go index b9aa8083..7c200781 100644 --- a/internal/query/queue/resultprivate_test.go +++ b/internal/query/queue/resultprivate_test.go @@ -34,7 +34,7 @@ func TestSetResult(t *testing.T) { textVersion: int32(2), } - qu := &queryprocessor.Query{ID: uuid.New(), Type: queryprocessor.TypeContextFull, RequiredQueryIDs: []uuid.UUID{}, Version: int32(1)} + qu := &queryprocessor.Query{ID: uuid.New(), Type: queryprocessor.TypeContextFull, Version: int32(1)} resultValues := []result.Value{} pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(qu.ID), database.MustToDBUUID(q.documentId), pgxmock.AnyArg(), q.cleanVersion, q.textVersion, qu.Version). @@ -42,7 +42,7 @@ func TestSetResult(t *testing.T) { pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), ) - err = q.setResult(ctx, qu, resultValues) + err = q.setResult(ctx, qu, &resultValues) assert.Nil(t, err) } diff --git a/internal/query/queue/service_test.go b/internal/query/queue/service_test.go index a273e261..4f03b4fb 100644 --- a/internal/query/queue/service_test.go +++ b/internal/query/queue/service_test.go @@ -12,6 +12,7 @@ import ( "testing" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) @@ -57,19 +58,22 @@ func TestService(t *testing.T) { contextID := uuid.New() contextVersion := int32(1) collectorQueries := []queryprocessor.Query{ - {ID: contextID, Type: queryprocessor.TypeContextFull, RequiredQueryIDs: []uuid.UUID{}, Version: contextVersion}, - {ID: queryOneID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{contextID}, Version: queryOneVersion}, - {ID: queryTwoID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{queryOneID}, Version: queryTwoVersion}, - {ID: queryThreeID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{queryOneID}, Version: queryThreeVersion}, - {ID: queryFourID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{contextID}, Version: queryFourVersion}, - {ID: queryFiveID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{querySixID}, Version: queryFiveVersion}, - {ID: querySixID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{contextID}, Version: querySixVersion}, + {ID: contextID, Type: queryprocessor.TypeContextFull, Version: contextVersion}, + {ID: queryOneID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: queryOneVersion}, + {ID: queryTwoID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryTwoVersion}, + {ID: queryThreeID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryThreeVersion}, + {ID: queryFourID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: queryFourVersion}, + {ID: queryFiveID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{querySixID}, Version: queryFiveVersion}, + {ID: querySixID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: querySixVersion}, } rows := pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}) for _, q := range collectorQueries { dbID := database.MustToDBUUID(q.ID) - dbReqIDs := database.MustToDBUUIDArray(q.RequiredQueryIDs) + dbReqIDs := []pgtype.UUID{} + if q.RequiredQueryIDs != nil { + dbReqIDs = database.MustToDBUUIDArray(*q.RequiredQueryIDs) + } ty, err := queryprocessor.ToDBNullQueryType(q.Type) assert.Nil(t, err) rows = rows. @@ -87,11 +91,11 @@ func TestService(t *testing.T) { } expectedQueries := []*queryprocessor.Query{ - {ID: querySixID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{contextID}, Version: querySixVersion}, - {ID: queryFiveID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{querySixID}, Version: queryFiveVersion}, - {ID: queryOneID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{contextID}, Version: queryOneVersion}, - {ID: queryThreeID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{queryOneID}, Version: queryThreeVersion}, - {ID: queryTwoID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: []uuid.UUID{queryOneID}, Version: queryTwoVersion}, + {ID: querySixID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: querySixVersion}, + {ID: queryFiveID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{querySixID}, Version: queryFiveVersion}, + {ID: queryOneID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: queryOneVersion}, + {ID: queryThreeID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryThreeVersion}, + {ID: queryTwoID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryTwoVersion}, } docID := uuid.New() diff --git a/internal/query/types/contextFull/creator_test.go b/internal/query/types/contextFull/creator_test.go index a7d6a794..1332c8e3 100644 --- a/internal/query/types/contextFull/creator_test.go +++ b/internal/query/types/contextFull/creator_test.go @@ -8,7 +8,6 @@ import ( contextfull "queryorchestration/internal/query/types/contextFull" "testing" - "github.com/google/uuid" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) @@ -30,9 +29,7 @@ func TestCreatorValidate(t *testing.T) { assert.NotNil(t, svc) entity := &queryprocessor.Create{ - Type: queryprocessor.TypeContextFull, - RequiredQueryIDs: []uuid.UUID{}, - Config: "", + Type: queryprocessor.TypeContextFull, } err = svc.Validate(ctx, entity) diff --git a/internal/query/types/contextFull/process_test.go b/internal/query/types/contextFull/process_test.go index b86b056e..f1738ae9 100644 --- a/internal/query/types/contextFull/process_test.go +++ b/internal/query/types/contextFull/process_test.go @@ -17,15 +17,14 @@ func TestContextFull(t *testing.T) { extractor := contextfull.NewExtractor() query := &queryprocessor.Query{ - ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, - RequiredQueryIDs: []uuid.UUID{}, - Version: int32(1), + ID: uuid.New(), + Type: queryprocessor.TypeJsonExtractor, + Version: int32(1), } values := []result.Value{} - value, err := extractor.Process(ctx, query, values) + value, err := extractor.Process(ctx, query, &values) assert.Nil(t, err) assert.Equal(t, "", value) @@ -33,6 +32,6 @@ func TestContextFull(t *testing.T) { contextfull.NewResult("example_result"), } - _, err = extractor.Process(ctx, query, values) + _, err = extractor.Process(ctx, query, &values) assert.EqualError(t, err, "no requirements expected") } diff --git a/internal/query/types/contextFull/service.go b/internal/query/types/contextFull/service.go index fec67137..1ae7d351 100644 --- a/internal/query/types/contextFull/service.go +++ b/internal/query/types/contextFull/service.go @@ -14,8 +14,8 @@ func NewExtractor() Extractor { return Extractor{} } -func (e Extractor) Process(ctx context.Context, query *queryprocessor.Query, values []result.Value) (string, error) { - if len(values) > 0 { +func (e Extractor) Process(ctx context.Context, query *queryprocessor.Query, values *[]result.Value) (string, error) { + if values != nil && len(*values) > 0 { return "", errors.New("no requirements expected") } // TODO diff --git a/internal/query/types/contextFull/updator_test.go b/internal/query/types/contextFull/updator_test.go index b6eb78d8..83a12b38 100644 --- a/internal/query/types/contextFull/updator_test.go +++ b/internal/query/types/contextFull/updator_test.go @@ -30,17 +30,13 @@ func TestUpdatorValidate(t *testing.T) { assert.NotNil(t, svc) current := &queryprocessor.Query{ - ID: uuid.New(), - Type: queryprocessor.TypeContextFull, - Version: int32(1), - RequiredQueryIDs: []uuid.UUID{}, - Config: "", + ID: uuid.New(), + Type: queryprocessor.TypeContextFull, + Version: int32(1), } entity := &queryprocessor.Update{ - ID: current.ID, - RequiredQueryIDs: []uuid.UUID{}, - Config: "", + ID: current.ID, } err = svc.Validate(ctx, current, entity) diff --git a/internal/query/types/jsonExtractor/creator_test.go b/internal/query/types/jsonExtractor/creator_test.go index 7697ac22..b7a12dff 100644 --- a/internal/query/types/jsonExtractor/creator_test.go +++ b/internal/query/types/jsonExtractor/creator_test.go @@ -8,7 +8,6 @@ import ( jsonextractor "queryorchestration/internal/query/types/jsonExtractor" "testing" - "github.com/google/uuid" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) @@ -30,9 +29,7 @@ func TestCreatorValidate(t *testing.T) { assert.NotNil(t, svc) entity := &queryprocessor.Create{ - Type: queryprocessor.TypeJsonExtractor, - RequiredQueryIDs: []uuid.UUID{}, - Config: "", + Type: queryprocessor.TypeJsonExtractor, } err = svc.Validate(ctx, entity) diff --git a/internal/query/types/jsonExtractor/process_test.go b/internal/query/types/jsonExtractor/process_test.go index b53a3adf..debeedf2 100644 --- a/internal/query/types/jsonExtractor/process_test.go +++ b/internal/query/types/jsonExtractor/process_test.go @@ -33,10 +33,9 @@ func TestJSONProcess(t *testing.T) { extractor := jsonextractor.NewExtractor(db) query := &queryprocessor.Query{ - ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, - RequiredQueryIDs: []uuid.UUID{}, - Version: int32(1), + ID: uuid.New(), + Type: queryprocessor.TypeJsonExtractor, + Version: int32(1), } entryValue := "value" @@ -53,7 +52,7 @@ func TestJSONProcess(t *testing.T) { AddRow(pgtype.UUID{}, []byte(config)), ) - value, err := extractor.Process(ctx, query, values) + value, err := extractor.Process(ctx, query, &values) assert.Nil(t, err) assert.Equal(t, entryValue, value) @@ -68,7 +67,7 @@ func TestJSONProcess(t *testing.T) { AddRow(pgtype.UUID{}, []byte(config)), ) - value, err = extractor.Process(ctx, query, values) + value, err = extractor.Process(ctx, query, &values) assert.Nil(t, err) assert.Equal(t, entryValue, value) @@ -83,7 +82,7 @@ func TestJSONProcess(t *testing.T) { AddRow(pgtype.UUID{}, []byte(config)), ) - value, err = extractor.Process(ctx, query, values) + value, err = extractor.Process(ctx, query, &values) assert.Nil(t, err) assert.Equal(t, entryValue, value) } @@ -104,10 +103,9 @@ func TestJSONProcessJSON(t *testing.T) { extractor := jsonextractor.NewExtractor(db) query := &queryprocessor.Query{ - ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, - RequiredQueryIDs: []uuid.UUID{}, - Version: int32(1), + ID: uuid.New(), + Type: queryprocessor.TypeJsonExtractor, + Version: int32(1), } entryValue := "value" @@ -124,7 +122,7 @@ func TestJSONProcessJSON(t *testing.T) { AddRow(pgtype.UUID{}, []byte(config)), ) - value, err := extractor.Process(ctx, query, values) + value, err := extractor.Process(ctx, query, &values) assert.EqualError(t, err, "JSON path does not exist: invalid_key") assert.Empty(t, value) @@ -136,7 +134,7 @@ func TestJSONProcessJSON(t *testing.T) { AddRow(pgtype.UUID{}, []byte(config)), ) - value, err = extractor.Process(ctx, query, values) + value, err = extractor.Process(ctx, query, &values) assert.EqualError(t, err, "unexpected end of JSON input") assert.Empty(t, value) @@ -148,7 +146,7 @@ func TestJSONProcessJSON(t *testing.T) { AddRow(pgtype.UUID{}, []byte(config)), ) - value, err = extractor.Process(ctx, query, values) + value, err = extractor.Process(ctx, query, &values) assert.EqualError(t, err, "JSON path does not exist: ") assert.Empty(t, value) @@ -160,7 +158,7 @@ func TestJSONProcessJSON(t *testing.T) { AddRow(pgtype.UUID{}, []byte(config)), ) - value, err = extractor.Process(ctx, query, values) + value, err = extractor.Process(ctx, query, &values) assert.EqualError(t, err, "invalid character '}' looking for beginning of value") assert.Empty(t, value) @@ -169,7 +167,7 @@ func TestJSONProcessJSON(t *testing.T) { pgxmock.NewRows([]string{"id", "config"}), ) - value, err = extractor.Process(ctx, query, values) + value, err = extractor.Process(ctx, query, &values) assert.EqualError(t, err, "no rows in result set") assert.Empty(t, value) } @@ -190,14 +188,13 @@ func TestJSONProcessResults(t *testing.T) { extractor := jsonextractor.NewExtractor(db) query := &queryprocessor.Query{ - ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, - RequiredQueryIDs: []uuid.UUID{}, - Version: int32(1), + ID: uuid.New(), + Type: queryprocessor.TypeJsonExtractor, + Version: int32(1), } results := []result.Value{} - value, err := extractor.Process(ctx, query, results) + value, err := extractor.Process(ctx, query, &results) assert.EqualError(t, err, "JSON Extraction requires 1 result") assert.Empty(t, value) @@ -205,7 +202,7 @@ func TestJSONProcessResults(t *testing.T) { contextfull.NewResult(""), contextfull.NewResult(""), } - value, err = extractor.Process(ctx, query, results) + value, err = extractor.Process(ctx, query, &results) assert.EqualError(t, err, "JSON Extraction requires 1 result") assert.Empty(t, value) @@ -214,7 +211,7 @@ func TestJSONProcessResults(t *testing.T) { contextfull.NewResult(""), contextfull.NewResult(""), } - value, err = extractor.Process(ctx, query, results) + value, err = extractor.Process(ctx, query, &results) assert.EqualError(t, err, "JSON Extraction requires 1 result") assert.Empty(t, value) } diff --git a/internal/query/types/jsonExtractor/service.go b/internal/query/types/jsonExtractor/service.go index f43cb6c1..928c0266 100644 --- a/internal/query/types/jsonExtractor/service.go +++ b/internal/query/types/jsonExtractor/service.go @@ -24,12 +24,12 @@ func NewExtractor(db *database.Connection) Extractor { return Extractor{db} } -func (e Extractor) Process(ctx context.Context, query *queryprocessor.Query, values []result.Value) (string, error) { - if len(values) != 1 { +func (e Extractor) Process(ctx context.Context, query *queryprocessor.Query, values *[]result.Value) (string, error) { + if values == nil || len(*values) != 1 { return "", fmt.Errorf("JSON Extraction requires 1 result") } - value, err := values[0].GetValue(ctx) + value, err := (*values)[0].GetValue(ctx) if err != nil { return "", err } diff --git a/internal/query/types/jsonExtractor/updator_test.go b/internal/query/types/jsonExtractor/updator_test.go index 280682a9..0d801d5c 100644 --- a/internal/query/types/jsonExtractor/updator_test.go +++ b/internal/query/types/jsonExtractor/updator_test.go @@ -30,17 +30,13 @@ func TestUpdatorValidate(t *testing.T) { assert.NotNil(t, svc) current := &queryprocessor.Query{ - ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, - Version: int32(1), - RequiredQueryIDs: []uuid.UUID{}, - Config: "", + ID: uuid.New(), + Type: queryprocessor.TypeJsonExtractor, + Version: int32(1), } entity := &queryprocessor.Update{ - ID: current.ID, - RequiredQueryIDs: []uuid.UUID{}, - Config: "", + ID: current.ID, } err = svc.Validate(ctx, current, entity) diff --git a/internal/query/update.go b/internal/query/update.go index 2c7713a2..404b0f98 100644 --- a/internal/query/update.go +++ b/internal/query/update.go @@ -3,9 +3,13 @@ package query import ( "context" "fmt" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" queryprocessor "queryorchestration/internal/query/processor" contextfull "queryorchestration/internal/query/types/contextFull" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" + + "github.com/google/uuid" ) func (s *Service) Update(ctx context.Context, entity *queryprocessor.Update) error { @@ -14,6 +18,35 @@ func (s *Service) Update(ctx context.Context, entity *queryprocessor.Update) err return err } + err = s.normalizeUpdate(ctx, current, entity) + if err != nil { + return err + } + + err = s.submitUpdate(ctx, current, entity) + if err != nil { + return err + } + + return nil +} + +func (s *Service) normalizeUpdate(ctx context.Context, current *Query, entity *queryprocessor.Update) error { + err := s.normalizeActiveVersion(current, entity) + if err != nil { + return err + } + + err = s.normalizeQueryIDs(ctx, entity) + if err != nil { + return err + } + + err = s.normalizeConfig(entity) + if err != nil { + return err + } + validator, err := s.getUpdator(current.Type) if err != nil { return err @@ -24,18 +57,123 @@ func (s *Service) Update(ctx context.Context, entity *queryprocessor.Update) err return err } - err = s.submitUpdate(ctx, entity) + return nil +} + +func (s *Service) submitUpdate(ctx context.Context, current *Query, entity *queryprocessor.Update) error { + tx, err := s.db.Pool.Begin(ctx) if err != nil { return err } + defer func() { + _ = tx.Rollback(ctx) + }() + + qtx := s.db.Queries.WithTx(tx) + + latestVersion := current.LatestVersion + 1 + id := database.MustToDBUUID(entity.ID) + + hasChanges := false + + addIDs := getSetDifference(entity.RequiredQueryIDs, current.RequiredQueryIDs) + for _, qID := range addIDs { + err = qtx.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ + Queryid: id, + Requiredqueryid: database.MustToDBUUID(qID), + Addedversion: latestVersion, + }) + if err != nil { + return err + } + hasChanges = true + } + + removeIDs := getSetDifference(current.RequiredQueryIDs, entity.RequiredQueryIDs) + for _, qID := range removeIDs { + err = qtx.RemoveRequiredQuery(ctx, &repository.RemoveRequiredQueryParams{ + Queryid: id, + Requiredqueryid: database.MustToDBUUID(qID), + Removedversion: &latestVersion, + }) + if err != nil { + return err + } + + hasChanges = true + } + + if entity.Config != nil && *entity.Config != "" { + err = qtx.RemoveQueryConfig(ctx, &repository.RemoveQueryConfigParams{ + Queryid: id, + Removedversion: &latestVersion, + }) + if err != nil { + return err + } + + err = qtx.AddQueryConfig(ctx, &repository.AddQueryConfigParams{ + Queryid: id, + Config: []byte(*entity.Config), + Addedversion: latestVersion, + }) + if err != nil { + return err + } + + hasChanges = true + } + + activeVersion := current.ActiveVersion + if entity.ActiveVersion != nil { + activeVersion = *entity.ActiveVersion + } + + if activeVersion != current.ActiveVersion { + hasChanges = true + } + + if hasChanges { + err = qtx.UpdateQuery(ctx, &repository.UpdateQueryParams{ + Latestversion: latestVersion, + Activeversion: activeVersion, + ID: id, + }) + if err != nil { + return err + } + + err = tx.Commit(ctx) + if err != nil { + return err + } + } return nil } -func (s *Service) submitUpdate(ctx context.Context, entity *queryprocessor.Update) error { - // TODO - generate new entity - // TODO - submit update - id, type, activeversion, requiredQueryId, Config - return nil +func getSetDifference(setA *[]uuid.UUID, setB *[]uuid.UUID) []uuid.UUID { + if setA == nil { + return []uuid.UUID{} + } else if setB == nil { + return *setA + } + + diff := []uuid.UUID{} + for _, q := range *setA { + isFound := false + for _, eq := range *setB { + if q == eq { + isFound = true + break + } + } + if !isFound { + diff = append(diff, q) + } + } + + return diff } func (s *Service) getUpdator(qType queryprocessor.Type) (queryprocessor.Updator, error) { diff --git a/internal/query/update_test.go b/internal/query/update_test.go index 6acfe263..0a598a08 100644 --- a/internal/query/update_test.go +++ b/internal/query/update_test.go @@ -9,6 +9,8 @@ import ( "testing" "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) @@ -30,29 +32,23 @@ func TestUpdate(t *testing.T) { config := "{\"path\":\"example_path\"}" existing := query.Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, ActiveVersion: int32(1), LatestVersion: int32(1), - RequiredQueryIDs: []uuid.UUID{ - uuid.New(), - }, - Config: config, } update := &queryprocessor.Update{ ID: existing.ID, - RequiredQueryIDs: []uuid.UUID{ - uuid.New(), - }, - Config: config, } - dbReqIDs := database.MustToDBUUIDArray(existing.RequiredQueryIDs) - pool.ExpectQuery("name: GetQuery :one").WithArgs(database.MustToDBUUID(update.ID)).WillReturnRows( pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). - AddRow(database.MustToDBUUID(existing.ID), repository.QuerytypeJsonExtractor, existing.ActiveVersion, existing.LatestVersion, []byte(config), dbReqIDs), + AddRow(database.MustToDBUUID(existing.ID), repository.QuerytypeJsonExtractor, existing.ActiveVersion, existing.LatestVersion, []byte(config), []pgtype.UUID{}), ) + pool.ExpectBeginTx(pgx.TxOptions{}) + pool.ExpectExec("name: UpdateQuery :exec").WithArgs(int32(1), int32(2), database.MustToDBUUID(update.ID)). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectCommit() + err = svc.Update(ctx, update) assert.Nil(t, err) } diff --git a/internal/query/updateprivate_test.go b/internal/query/updateprivate_test.go index eb7b4dcb..2fb1f997 100644 --- a/internal/query/updateprivate_test.go +++ b/internal/query/updateprivate_test.go @@ -2,12 +2,14 @@ package query import ( "context" + "errors" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" queryprocessor "queryorchestration/internal/query/processor" "testing" "github.com/google/uuid" + "github.com/jackc/pgx/v5" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) @@ -56,17 +58,246 @@ func TestSubmitUpdate(t *testing.T) { config := "{\"path\":\"example_path\"}" q := Query{ ID: uuid.New(), - RequiredQueryIDs: []uuid.UUID{ + RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, - Config: config, + Config: &config, + ActiveVersion: int32(1), + LatestVersion: int32(2), } + aV := int32(10) update := &queryprocessor.Update{ - ID: q.ID, - RequiredQueryIDs: q.RequiredQueryIDs, - Config: q.Config, + ID: q.ID, + RequiredQueryIDs: &[]uuid.UUID{ + uuid.New(), + }, + Config: q.Config, + ActiveVersion: &aV, } - err = svc.submitUpdate(ctx, update) + pool.ExpectBeginTx(pgx.TxOptions{}) + pool.ExpectExec("name: AddRequiredQuery :exec").WithArgs(database.MustToDBUUID(update.ID), database.MustToDBUUID((*update.RequiredQueryIDs)[0]), pgxmock.AnyArg()). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectExec("name: RemoveRequiredQuery :exec").WithArgs(pgxmock.AnyArg(), database.MustToDBUUID((*q.RequiredQueryIDs)[0]), database.MustToDBUUID(update.ID)). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectExec("name: RemoveQueryConfig :exec").WithArgs(pgxmock.AnyArg(), database.MustToDBUUID(update.ID)). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectExec("name: AddQueryConfig :exec").WithArgs(database.MustToDBUUID(update.ID), []byte(*update.Config), int32(3)). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectExec("name: UpdateQuery :exec").WithArgs(aV, int32(3), database.MustToDBUUID(update.ID)). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectCommit() + + err = svc.submitUpdate(ctx, &q, update) assert.Nil(t, err) } + +func TestSubmitUpdateRollback(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := New(db) + + q := Query{ + ID: uuid.New(), + ActiveVersion: int32(1), + LatestVersion: int32(2), + } + aV := int32(10) + update := &queryprocessor.Update{ + ID: q.ID, + ActiveVersion: &aV, + } + + pool.ExpectBeginTx(pgx.TxOptions{}) + msg := "database failure" + pool.ExpectExec("name: UpdateQuery :exec").WithArgs(aV, int32(3), database.MustToDBUUID(update.ID)). + WillReturnError(errors.New(msg)) + pool.ExpectCommit() + + err = svc.submitUpdate(ctx, &q, update) + assert.EqualError(t, err, msg) +} + +func TestSubmitUpdateRequiredQueries(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := New(db) + + config := "{\"path\":\"example_path\"}" + q := Query{ + ID: uuid.New(), + RequiredQueryIDs: &[]uuid.UUID{ + uuid.New(), + uuid.New(), + uuid.New(), + }, + Config: &config, + ActiveVersion: int32(1), + LatestVersion: int32(2), + } + update := &queryprocessor.Update{ + ID: q.ID, + RequiredQueryIDs: &[]uuid.UUID{ + uuid.New(), + uuid.New(), + (*q.RequiredQueryIDs)[0], + }, + } + + pool.ExpectBeginTx(pgx.TxOptions{}) + pool.ExpectExec("name: AddRequiredQuery :exec").WithArgs(database.MustToDBUUID(update.ID), database.MustToDBUUID((*update.RequiredQueryIDs)[0]), pgxmock.AnyArg()). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectExec("name: AddRequiredQuery :exec").WithArgs(database.MustToDBUUID(update.ID), database.MustToDBUUID((*update.RequiredQueryIDs)[1]), pgxmock.AnyArg()). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectExec("name: RemoveRequiredQuery :exec").WithArgs(pgxmock.AnyArg(), database.MustToDBUUID((*q.RequiredQueryIDs)[1]), database.MustToDBUUID(update.ID)). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectExec("name: RemoveRequiredQuery :exec").WithArgs(pgxmock.AnyArg(), database.MustToDBUUID((*q.RequiredQueryIDs)[2]), database.MustToDBUUID(update.ID)). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectExec("name: UpdateQuery :exec").WithArgs(q.ActiveVersion, int32(3), database.MustToDBUUID(update.ID)). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectCommit() + + err = svc.submitUpdate(ctx, &q, update) + assert.Nil(t, err) +} + +func TestSubmitUpdateActiveVersion(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := New(db) + + q := Query{ + ID: uuid.New(), + ActiveVersion: int32(1), + LatestVersion: int32(2), + } + aV := int32(10) + update := &queryprocessor.Update{ + ID: q.ID, + ActiveVersion: &aV, + } + + pool.ExpectBeginTx(pgx.TxOptions{}) + pool.ExpectExec("name: UpdateQuery :exec").WithArgs(aV, int32(3), database.MustToDBUUID(update.ID)). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectCommit() + + err = svc.submitUpdate(ctx, &q, update) + assert.Nil(t, err) +} + +func TestSubmitUpdateNoChange(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := New(db) + + q := Query{ + ID: uuid.New(), + ActiveVersion: int32(1), + LatestVersion: int32(2), + } + update := &queryprocessor.Update{ + ID: q.ID, + } + + pool.ExpectBeginTx(pgx.TxOptions{}) + pool.ExpectRollback() + + err = svc.submitUpdate(ctx, &q, update) + assert.Nil(t, err) +} + +func TestGetSetDifference(t *testing.T) { + commonUUID := uuid.New() + listA := []uuid.UUID{uuid.New(), commonUUID} + listB := []uuid.UUID{uuid.New(), commonUUID} + + assert.ElementsMatch(t, []uuid.UUID{}, getSetDifference(nil, nil)) + assert.ElementsMatch(t, listA, getSetDifference(&listA, nil)) + assert.ElementsMatch(t, []uuid.UUID{}, getSetDifference(nil, &listB)) + assert.ElementsMatch(t, []uuid.UUID{listA[0]}, getSetDifference(&listA, &listB)) + assert.ElementsMatch(t, []uuid.UUID{listB[0]}, getSetDifference(&listB, &listA)) + assert.ElementsMatch(t, []uuid.UUID{}, getSetDifference(&listA, &listA)) +} + +func TestNormalizeUpdate(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := New(db) + + current := &Query{ + ID: uuid.New(), + ActiveVersion: int32(1), + LatestVersion: int32(2), + Type: queryprocessor.TypeJsonExtractor, + } + cfg := "{}" + aV := int32(2) + update := &queryprocessor.Update{ + ID: current.ID, + Config: &cfg, + ActiveVersion: &aV, + RequiredQueryIDs: &[]uuid.UUID{}, + } + + dbids := database.MustToDBUUIDArray(*update.RequiredQueryIDs) + + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(true), + ) + + err = svc.normalizeUpdate(ctx, current, update) + assert.Nil(t, err) + assert.EqualExportedValues(t, queryprocessor.Update{ + ID: current.ID, + ActiveVersion: &aV, + Config: nil, + RequiredQueryIDs: nil, + }, *update) +} diff --git a/pkg/queryService/api.gen.go b/pkg/queryService/api.gen.go index 84558802..63d734fc 100644 --- a/pkg/queryService/api.gen.go +++ b/pkg/queryService/api.gen.go @@ -134,7 +134,7 @@ type Query struct { LatestVersion int32 `json:"latest_version"` // RequiredQueries List of required query IDs. - RequiredQueries []string `json:"required_queries"` + RequiredQueries *[]string `json:"required_queries,omitempty"` // Type Specifies the type of the query. Type QueryType `json:"type"` diff --git a/scripts/Taskfile.yml b/scripts/Taskfile.yml index 60366109..e60718f5 100644 --- a/scripts/Taskfile.yml +++ b/scripts/Taskfile.yml @@ -33,6 +33,7 @@ tasks: - touch .env fullsuite: cmds: + - task deps:tidy - task build - task lint - task test:unit diff --git a/serviceAPIs/queryService.yaml b/serviceAPIs/queryService.yaml index 06a22387..37b1e58b 100644 --- a/serviceAPIs/queryService.yaml +++ b/serviceAPIs/queryService.yaml @@ -318,7 +318,6 @@ components: - type - active_version - latest_version - - required_queries ListQueries: type: object diff --git a/test/queryService/queryservice_test.go b/test/queryService/queryservice_test.go index 072d656c..d4a27e40 100644 --- a/test/queryService/queryservice_test.go +++ b/test/queryService/queryservice_test.go @@ -24,13 +24,19 @@ func TestQueryService(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, idRes) assert.NotNil(t, idRes.JSON201) - id := idRes.JSON201.Id - assert.NotEmpty(t, id) + contextID := idRes.JSON201.Id + assert.NotEmpty(t, contextID) - queryRes, err := client.GetQueryByIdWithResponse(ctx, id) + idRes, err = client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{ + Type: queryservice.JSONEXTRACTOR, + }) assert.Nil(t, err) - assert.Equal(t, id, queryRes.JSON200.Id) - assert.Equal(t, queryservice.CONTEXTFULL, queryRes.JSON200.Type) + jsonID := idRes.JSON201.Id + + queryRes, err := client.GetQueryByIdWithResponse(ctx, jsonID) + assert.Nil(t, err) + assert.Equal(t, jsonID, queryRes.JSON200.Id) + assert.Equal(t, queryservice.JSONEXTRACTOR, queryRes.JSON200.Type) assert.Equal(t, int32(1), queryRes.JSON200.ActiveVersion) assert.Equal(t, int32(1), queryRes.JSON200.LatestVersion) assert.Nil(t, queryRes.JSON200.Config) @@ -38,23 +44,26 @@ func TestQueryService(t *testing.T) { queriesRes, err := client.ListQueriesWithResponse(ctx) assert.Nil(t, err) - assert.Len(t, queriesRes.JSON200.Queries, 1) - assert.Equal(t, id, queriesRes.JSON200.Queries[0].Id) + assert.Len(t, queriesRes.JSON200.Queries, 2) - res, err := client.UpdateQueryWithResponse(ctx, id, queryservice.QueryUpdate{ - Config: nil, - ActiveVersion: nil, - RequiredQueries: &[]string{}, + cfg := "{}" + aV := int32(2) + res, err := client.UpdateQueryWithResponse(ctx, jsonID, queryservice.QueryUpdate{ + Config: &cfg, + ActiveVersion: &aV, + RequiredQueries: &[]string{ + contextID, + }, }) assert.Nil(t, err) assert.NotNil(t, res) - queryRes, err = client.GetQueryByIdWithResponse(ctx, id) + queryRes, err = client.GetQueryByIdWithResponse(ctx, jsonID) assert.Nil(t, err) - assert.Equal(t, id, queryRes.JSON200.Id) - assert.Equal(t, queryservice.CONTEXTFULL, queryRes.JSON200.Type) - assert.Equal(t, int32(1), queryRes.JSON200.ActiveVersion) - assert.Equal(t, int32(1), queryRes.JSON200.LatestVersion) + assert.Equal(t, jsonID, queryRes.JSON200.Id) + assert.Equal(t, queryservice.JSONEXTRACTOR, queryRes.JSON200.Type) + assert.Equal(t, int32(2), queryRes.JSON200.ActiveVersion) + assert.Equal(t, int32(2), queryRes.JSON200.LatestVersion) assert.Nil(t, queryRes.JSON200.Config) - assert.Nil(t, queryRes.JSON200.RequiredQueries) + assert.ElementsMatch(t, []string{contextID}, *queryRes.JSON200.RequiredQueries) }