jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/listtypes (pull request #23)

Browse Source 
Feature/listtypes

* started

* cleanuplist

* someunittestfixes

* removemostflakiness
This commit is contained in:
Michael McGuinness
2025-01-15 12:19:49 +00:00
parent 7da57058d4
commit 5ca36b0502
38 changed files with 532 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/api/query.go
+1 -8
View File
@@ -29,14 +29,7 @@ func NewQueryController(querySvc query.Service, validator *validator.Validate) *
}
func (s *QueryController) List(ctx context.Context, req *serviceinterfaces.QueryFilter) (*serviceinterfaces.Queries, error) {
types := make([]queryprocessor.Type, len(req.GetTypes()))
for index, t := range req.GetTypes() {
types[index] = queryprocessor.Type(ParseSpecQueryType(t))
}
filters := query.ListFilters{
Types: types,
}
filters := query.ListFilters{}
queries, err := s.query.List(ctx, filters)
if err != nil {
api/queue/queryRunner_test.go
+3 -2
View File
@@ -62,6 +62,7 @@ func TestQueryRunner(t *testing.T) {
URL: qCfg.URL,
Client: qCfg.Client,
}
qV := int32(1)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)).
WillReturnRows(
@@ -71,12 +72,12 @@ func TestQueryRunner(t *testing.T) {
pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}).
AddRow(collectorID, queryID, int32(1)),
AddRow(collectorID, queryID, qV),
)
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(collectorID).
WillReturnRows(
pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
AddRow(collectorID, queryID, repository.NullQuerytype{Querytype: repository.QuerytypeContextFull, Valid: true}, pgtype.Int4{Int32: int32(1), Valid: true}, []pgtype.UUID{}),
AddRow(collectorID, queryID, repository.NullQuerytype{Querytype: repository.QuerytypeContextFull, Valid: true}, &qV, []pgtype.UUID{}),
)
err = runner.Process(ctx, cfg, msg)
api/serviceInterfaces/main.pb.go
+114 -127
View File
@@ -265,8 +265,6 @@ type QueryFilter struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Types []QueryType `protobuf:"varint,1,rep,packed,name=types,proto3,enum=queryorchestration.QueryType" json:"types,omitempty"`
}
func (x *QueryFilter) Reset() {
@@ -299,13 +297,6 @@ func (*QueryFilter) Descriptor() ([]byte, []int) {
return file_main_proto_rawDescGZIP(), []int{3}
}
func (x *QueryFilter) GetTypes() []QueryType {
if x != nil {
return x.Types
}
return nil
}
type QueryCreate struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -701,100 +692,97 @@ var file_main_proto_rawDesc = []byte{
0x69, 0x67, 0x88, 0x01, 0x01, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65,
0x64, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52,
0x0f, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x51, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73,
0x42, 0x09, 0x0a, 0x07, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x42, 0x0a, 0x0b, 0x51,
0x75, 0x65, 0x72, 0x79, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x33, 0x0a, 0x05, 0x74, 0x79,
0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x71, 0x75, 0x65, 0x72,
0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x51,
0x75, 0x65, 0x72, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x22,
0x93, 0x01, 0x0a, 0x0b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12,
0x31, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e,
0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79,
0x70, 0x65, 0x12, 0x1b, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01,
0x28, 0x09, 0x48, 0x00, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x88, 0x01, 0x01, 0x12,
0x29, 0x0a, 0x10, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x5f, 0x71, 0x75, 0x65, 0x72,
0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x69,
0x72, 0x65, 0x64, 0x51, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x63,
0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0xaf, 0x01, 0x0a, 0x0b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x55,
0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1b, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x88,
0x01, 0x01, 0x12, 0x2a, 0x0a, 0x0e, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x76, 0x65, 0x72,
0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x48, 0x01, 0x52, 0x0d, 0x61, 0x63,
0x74, 0x69, 0x76, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x29,
0x0a, 0x10, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x69,
0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72,
0x65, 0x64, 0x51, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x63, 0x6f,
0x6e, 0x66, 0x69, 0x67, 0x42, 0x11, 0x0a, 0x0f, 0x5f, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f,
0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x3e, 0x0a, 0x07, 0x51, 0x75, 0x65, 0x72, 0x69,
0x65, 0x73, 0x12, 0x33, 0x0a, 0x07, 0x71, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20,
0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65,
0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x07,
0x71, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73, 0x22, 0x0e, 0x0a, 0x0c, 0x4a, 0x6f, 0x62, 0x43, 0x6f,
0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x22, 0x14, 0x0a, 0x12, 0x4a, 0x6f, 0x62, 0x43, 0x6f,
0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x22, 0x14, 0x0a,
0x12, 0x4a, 0x6f, 0x62, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x55, 0x70, 0x64,
0x61, 0x74, 0x65, 0x22, 0x0f, 0x0a, 0x0d, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x54, 0x72, 0x69,
0x67, 0x67, 0x65, 0x72, 0x22, 0x1b, 0x0a, 0x09, 0x49, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67,
0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69,
0x64, 0x2a, 0x63, 0x0a, 0x09, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a,
0x0a, 0x16, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53,
0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x51, 0x55,
0x45, 0x52, 0x59, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x58,
0x54, 0x52, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x51, 0x55, 0x45,
0x52, 0x59, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x45, 0x58, 0x54, 0x5f,
0x46, 0x55, 0x4c, 0x4c, 0x10, 0x02, 0x32, 0xbb, 0x03, 0x0a, 0x0c, 0x51, 0x75, 0x65, 0x72, 0x79,
0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x44, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12,
0x1f, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61,
0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72,
0x1a, 0x1b, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73, 0x12, 0x3f, 0x0a,
0x03, 0x47, 0x65, 0x74, 0x12, 0x1d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68,
0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x49, 0x64, 0x4d, 0x65, 0x73, 0x73,
0x61, 0x67, 0x65, 0x1a, 0x19, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65,
0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x48,
0x0a, 0x06, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79,
0x42, 0x09, 0x0a, 0x07, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x0d, 0x0a, 0x0b, 0x51,
0x75, 0x65, 0x72, 0x79, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x22, 0x93, 0x01, 0x0a, 0x0b, 0x51,
0x75, 0x65, 0x72, 0x79, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x31, 0x0a, 0x04, 0x74, 0x79,
0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79,
0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x51, 0x75,
0x65, 0x72, 0x79, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x1a, 0x1d, 0x2e, 0x71, 0x75, 0x65, 0x72,
0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x49,
0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x41, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61,
0x74, 0x65, 0x12, 0x1f, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73,
0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x55, 0x70, 0x64,
0x61, 0x74, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x42, 0x0a, 0x09, 0x44,
0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79,
0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x49, 0x64,
0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12,
0x53, 0x0a, 0x04, 0x54, 0x65, 0x73, 0x74, 0x12, 0x24, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f,
0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x51, 0x75, 0x65,
0x72, 0x79, 0x54, 0x65, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e,
0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x32, 0xf8, 0x01, 0x0a, 0x13, 0x4a, 0x6f, 0x62, 0x43, 0x6f, 0x6c, 0x6c,
0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4f, 0x0a, 0x06,
0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x26, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72,
0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4a, 0x6f, 0x62, 0x43,
0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x1a, 0x1d,
0x65, 0x72, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a,
0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52,
0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x88, 0x01, 0x01, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65,
0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73, 0x18, 0x03,
0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x51, 0x75,
0x65, 0x72, 0x69, 0x65, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67,
0x22, 0xaf, 0x01, 0x0a, 0x0b, 0x51, 0x75, 0x65, 0x72, 0x79, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65,
0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64,
0x12, 0x1b, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
0x48, 0x00, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x88, 0x01, 0x01, 0x12, 0x2a, 0x0a,
0x0e, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18,
0x03, 0x20, 0x01, 0x28, 0x05, 0x48, 0x01, 0x52, 0x0d, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x56,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x71,
0x75, 0x69, 0x72, 0x65, 0x64, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20,
0x03, 0x28, 0x09, 0x52, 0x0f, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x51, 0x75, 0x65,
0x72, 0x69, 0x65, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42,
0x11, 0x0a, 0x0f, 0x5f, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69,
0x6f, 0x6e, 0x22, 0x3e, 0x0a, 0x07, 0x51, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73, 0x12, 0x33, 0x0a,
0x07, 0x71, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19,
0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x2e, 0x49, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x48, 0x0a,
0x06, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x26, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f,
0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4a, 0x6f, 0x62,
0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x1a,
0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75,
0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x46, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x1d,
0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x2e, 0x49, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x20, 0x2e,
0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x2e, 0x4a, 0x6f, 0x62, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x32,
0x5c, 0x0a, 0x0d, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
0x12, 0x4b, 0x0a, 0x07, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x12, 0x21, 0x2e, 0x71, 0x75,
0x69, 0x6f, 0x6e, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x07, 0x71, 0x75, 0x65, 0x72, 0x69,
0x65, 0x73, 0x22, 0x0e, 0x0a, 0x0c, 0x4a, 0x6f, 0x62, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x22, 0x14, 0x0a, 0x12, 0x4a, 0x6f, 0x62, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x22, 0x14, 0x0a, 0x12, 0x4a, 0x6f, 0x62, 0x43,
0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x22, 0x0f,
0x0a, 0x0d, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x22,
0x1b, 0x0a, 0x09, 0x49, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x0e, 0x0a, 0x02,
0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x2a, 0x63, 0x0a, 0x09,
0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x16, 0x51, 0x55, 0x45,
0x52, 0x59, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46,
0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x54,
0x59, 0x50, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x45, 0x58, 0x54, 0x52, 0x41, 0x43, 0x54,
0x4f, 0x52, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x51, 0x55, 0x45, 0x52, 0x59, 0x5f, 0x54, 0x59,
0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x45, 0x58, 0x54, 0x5f, 0x46, 0x55, 0x4c, 0x4c, 0x10,
0x02, 0x32, 0xbb, 0x03, 0x0a, 0x0c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x12, 0x44, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1f, 0x2e, 0x71, 0x75, 0x65,
0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e,
0x51, 0x75, 0x65, 0x72, 0x79, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x1a, 0x1b, 0x2e, 0x71, 0x75,
0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x1a, 0x1d,
0x2e, 0x51, 0x75, 0x65, 0x72, 0x69, 0x65, 0x73, 0x12, 0x3f, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12,
0x1d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61,
0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x49, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x19,
0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x2e, 0x49, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x15, 0x5a,
0x13, 0x2e, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x66,
0x61, 0x63, 0x65, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x69, 0x6f, 0x6e, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x48, 0x0a, 0x06, 0x43, 0x72, 0x65,
0x61, 0x74, 0x65, 0x12, 0x1f, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65,
0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x43, 0x72,
0x65, 0x61, 0x74, 0x65, 0x1a, 0x1d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68,
0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x49, 0x64, 0x4d, 0x65, 0x73, 0x73,
0x61, 0x67, 0x65, 0x12, 0x41, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x2e,
0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x1a, 0x16,
0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x42, 0x0a, 0x09, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63,
0x61, 0x74, 0x65, 0x12, 0x1d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65,
0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x49, 0x64, 0x4d, 0x65, 0x73, 0x73, 0x61,
0x67, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x53, 0x0a, 0x04, 0x54, 0x65,
0x73, 0x74, 0x12, 0x24, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73,
0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x54, 0x65, 0x73,
0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79,
0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x51, 0x75,
0x65, 0x72, 0x79, 0x54, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32,
0xf8, 0x01, 0x0a, 0x13, 0x4a, 0x6f, 0x62, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72,
0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4f, 0x0a, 0x06, 0x43, 0x72, 0x65, 0x61, 0x74,
0x65, 0x12, 0x26, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4a, 0x6f, 0x62, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63,
0x74, 0x6f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x1a, 0x1d, 0x2e, 0x71, 0x75, 0x65, 0x72,
0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x49,
0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x48, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61,
0x74, 0x65, 0x12, 0x26, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73,
0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4a, 0x6f, 0x62, 0x43, 0x6f, 0x6c, 0x6c, 0x65,
0x63, 0x74, 0x6f, 0x72, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f,
0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70,
0x74, 0x79, 0x12, 0x46, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x1d, 0x2e, 0x71, 0x75, 0x65, 0x72,
0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x49,
0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x20, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79,
0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4a, 0x6f,
0x62, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x32, 0x5c, 0x0a, 0x0d, 0x45, 0x78,
0x70, 0x6f, 0x72, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4b, 0x0a, 0x07, 0x54,
0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x12, 0x21, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x6f, 0x72,
0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x78, 0x70, 0x6f,
0x72, 0x74, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x1a, 0x1d, 0x2e, 0x71, 0x75, 0x65, 0x72,
0x79, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x49,
0x64, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x15, 0x5a, 0x13, 0x2e, 0x2f, 0x73, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61, 0x63, 0x65, 0x73, 0x62,
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -829,34 +817,33 @@ var file_main_proto_goTypes = []any{
}
var file_main_proto_depIdxs = []int32{
0, // 0: queryorchestration.Query.type:type_name -> queryorchestration.QueryType
0, // 1: queryorchestration.QueryFilter.types:type_name -> queryorchestration.QueryType
0, // 2: queryorchestration.QueryCreate.type:type_name -> queryorchestration.QueryType
3, // 3: queryorchestration.Queries.queries:type_name -> queryorchestration.Query
4, // 4: queryorchestration.QueryService.List:input_type -> queryorchestration.QueryFilter
12, // 5: queryorchestration.QueryService.Get:input_type -> queryorchestration.IdMessage
5, // 6: queryorchestration.QueryService.Create:input_type -> queryorchestration.QueryCreate
6, // 7: queryorchestration.QueryService.Update:input_type -> queryorchestration.QueryUpdate
12, // 8: queryorchestration.QueryService.Deprecate:input_type -> queryorchestration.IdMessage
1, // 9: queryorchestration.QueryService.Test:input_type -> queryorchestration.QueryTestRequest
9, // 10: queryorchestration.JobCollectorService.Create:input_type -> queryorchestration.JobCollectorCreate
10, // 11: queryorchestration.JobCollectorService.Update:input_type -> queryorchestration.JobCollectorUpdate
12, // 12: queryorchestration.JobCollectorService.Get:input_type -> queryorchestration.IdMessage
11, // 13: queryorchestration.ExportService.Trigger:input_type -> queryorchestration.ExportTrigger
7, // 14: queryorchestration.QueryService.List:output_type -> queryorchestration.Queries
3, // 15: queryorchestration.QueryService.Get:output_type -> queryorchestration.Query
12, // 16: queryorchestration.QueryService.Create:output_type -> queryorchestration.IdMessage
13, // 17: queryorchestration.QueryService.Update:output_type -> google.protobuf.Empty
13, // 18: queryorchestration.QueryService.Deprecate:output_type -> google.protobuf.Empty
2, // 19: queryorchestration.QueryService.Test:output_type -> queryorchestration.QueryTestResponse
12, // 20: queryorchestration.JobCollectorService.Create:output_type -> queryorchestration.IdMessage
13, // 21: queryorchestration.JobCollectorService.Update:output_type -> google.protobuf.Empty
8, // 22: queryorchestration.JobCollectorService.Get:output_type -> queryorchestration.JobCollector
12, // 23: queryorchestration.ExportService.Trigger:output_type -> queryorchestration.IdMessage
14, // [14:24] is the sub-list for method output_type
4, // [4:14] is the sub-list for method input_type
4, // [4:4] is the sub-list for extension type_name
4, // [4:4] is the sub-list for extension extendee
0, // [0:4] is the sub-list for field type_name
0, // 1: queryorchestration.QueryCreate.type:type_name -> queryorchestration.QueryType
3, // 2: queryorchestration.Queries.queries:type_name -> queryorchestration.Query
4, // 3: queryorchestration.QueryService.List:input_type -> queryorchestration.QueryFilter
12, // 4: queryorchestration.QueryService.Get:input_type -> queryorchestration.IdMessage
5, // 5: queryorchestration.QueryService.Create:input_type -> queryorchestration.QueryCreate
6, // 6: queryorchestration.QueryService.Update:input_type -> queryorchestration.QueryUpdate
12, // 7: queryorchestration.QueryService.Deprecate:input_type -> queryorchestration.IdMessage
1, // 8: queryorchestration.QueryService.Test:input_type -> queryorchestration.QueryTestRequest
9, // 9: queryorchestration.JobCollectorService.Create:input_type -> queryorchestration.JobCollectorCreate
10, // 10: queryorchestration.JobCollectorService.Update:input_type -> queryorchestration.JobCollectorUpdate
12, // 11: queryorchestration.JobCollectorService.Get:input_type -> queryorchestration.IdMessage
11, // 12: queryorchestration.ExportService.Trigger:input_type -> queryorchestration.ExportTrigger
7, // 13: queryorchestration.QueryService.List:output_type -> queryorchestration.Queries
3, // 14: queryorchestration.QueryService.Get:output_type -> queryorchestration.Query
12, // 15: queryorchestration.QueryService.Create:output_type -> queryorchestration.IdMessage
13, // 16: queryorchestration.QueryService.Update:output_type -> google.protobuf.Empty
13, // 17: queryorchestration.QueryService.Deprecate:output_type -> google.protobuf.Empty
2, // 18: queryorchestration.QueryService.Test:output_type -> queryorchestration.QueryTestResponse
12, // 19: queryorchestration.JobCollectorService.Create:output_type -> queryorchestration.IdMessage
13, // 20: queryorchestration.JobCollectorService.Update:output_type -> google.protobuf.Empty
8, // 21: queryorchestration.JobCollectorService.Get:output_type -> queryorchestration.JobCollector
12, // 22: queryorchestration.ExportService.Trigger:output_type -> queryorchestration.IdMessage
13, // [13:23] is the sub-list for method output_type
3, // [3:13] is the sub-list for method input_type
3, // [3:3] is the sub-list for extension type_name
3, // [3:3] is the sub-list for extension extendee
0, // [0:3] is the sub-list for field type_name
}
func init() { file_main_proto_init() }
internal/database/connection.go
+15 -11
View File
@@ -71,6 +71,21 @@ func getPoolConfig() (*pgxpool.Config, error) {
return nil, err
}
enumTypes := []string{
"querytype",
}
config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
for _, enumName := range enumTypes {
dt, err := conn.LoadType(ctx, enumName)
if err != nil {
return fmt.Errorf("failed to load suitable enum type: %w", err)
}
conn.TypeMap().RegisterType(dt)
}
return nil
}
return config, nil
}
@@ -93,14 +108,3 @@ func GetDBPool(ctx context.Context) *pgxpool.Pool {
return pool
}
func GetDBConn(ctx context.Context) *pgx.Conn {
config := mustGetURI()
conn, err := pgx.Connect(ctx, config.String())
if err != nil {
log.Panicf("Unable to connect to database: %v\n", err)
}
return conn
}
internal/database/connection_test.go
-14
View File
@@ -20,20 +20,6 @@ func TestDBConn(t *testing.T) {
})
defer cleanup()
conn := database.GetDBConn(ctx)
assert.NotNil(t, conn)
pool := database.GetDBPool(ctx)
assert.NotNil(t, pool)
}
func TestDBConnNoDB(t *testing.T) {
ctx := context.Background()
t.Setenv("DB_USER", "user")
t.Setenv("DB_PASS", "pass")
t.Setenv("DB_HOST", "host")
t.Setenv("DB_PORT", "5432")
t.Setenv("DB_NAME", "name")
assert.Panics(t, func() { database.GetDBConn(ctx) })
}
internal/database/migrations_test.go
+1 -3
View File
@@ -14,9 +14,7 @@ import (
func TestRunMigrations(t *testing.T) {
ctx := context.Background()
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
NoPool: true,
})
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{})
defer cleanup()
database.RunMigrations(ctx, &database.MigrationConfig{
internal/database/repository/collector.sql.go
+23 -14
View File
@@ -16,13 +16,16 @@ SELECT id, jobId, minCleanVersion, minTextVersion FROM collectors WHERE jobId =
`
type GetCollectorFromJobIDRow struct {
ID pgtype.UUID
Jobid pgtype.UUID
Mincleanversion int32
Mintextversion int32
ID pgtype.UUID `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
}
func (q *Queries) GetCollectorFromJobID(ctx context.Context, jobid pgtype.UUID) (GetCollectorFromJobIDRow, error) {
// GetCollectorFromJobID
//
// SELECT id, jobId, minCleanVersion, minTextVersion FROM collectors WHERE jobId = $1 LIMIT 1
func (q *Queries) GetCollectorFromJobID(ctx context.Context, jobid pgtype.UUID) (*GetCollectorFromJobIDRow, error) {
row := q.db.QueryRow(ctx, getCollectorFromJobID, jobid)
var i GetCollectorFromJobIDRow
err := row.Scan(
@@ -31,7 +34,7 @@ func (q *Queries) GetCollectorFromJobID(ctx context.Context, jobid pgtype.UUID)
&i.Mincleanversion,
&i.Mintextversion,
)
return i, err
return &i, err
}
const getCollectorQueries = `-- name: GetCollectorQueries :many
@@ -42,20 +45,26 @@ SELECT collectorId, queryId, type, queryVersion, ARRAY_AGG(requiredQueryId) AS r
`
type GetCollectorQueriesRow struct {
Collectorid pgtype.UUID
Queryid pgtype.UUID
Type NullQuerytype
Queryversion pgtype.Int4
Requiredids []pgtype.UUID
Collectorid pgtype.UUID `db:"collectorid"`
Queryid pgtype.UUID `db:"queryid"`
Type NullQuerytype `db:"type"`
Queryversion *int32 `db:"queryversion"`
Requiredids []pgtype.UUID `db:"requiredids"`
}
func (q *Queries) GetCollectorQueries(ctx context.Context, collectorid pgtype.UUID) ([]GetCollectorQueriesRow, error) {
// GetCollectorQueries
//
// SELECT collectorId, queryId, type, queryVersion, ARRAY_AGG(requiredQueryId) AS requiredIds
// FROM collectorQueryDependencyTree
// WHERE collectorId = $1
// GROUP BY queryId, collectorId, type, queryVersion
func (q *Queries) GetCollectorQueries(ctx context.Context, collectorid pgtype.UUID) ([]*GetCollectorQueriesRow, error) {
rows, err := q.db.Query(ctx, getCollectorQueries, collectorid)
if err != nil {
return nil, err
}
defer rows.Close()
var items []GetCollectorQueriesRow
items := []*GetCollectorQueriesRow{}
for rows.Next() {
var i GetCollectorQueriesRow
if err := rows.Scan(
@@ -67,7 +76,7 @@ func (q *Queries) GetCollectorQueries(ctx context.Context, collectorid pgtype.UU
); err != nil {
return nil, err
}
items = append(items, i)
items = append(items, &i)
}
if err := rows.Err(); err != nil {
return nil, err
internal/database/repository/models.go
+64 -55
View File
@@ -53,90 +53,99 @@ func (ns NullQuerytype) Value() (driver.Value, error) {
return string(ns.Querytype), nil
}
func (e Querytype) Valid() bool {
switch e {
case QuerytypeContextFull,
QuerytypeJsonExtractor:
return true
}
return false
}
type Activecollectorquery struct {
Collectorid pgtype.UUID
Activeversion int32
Queryid pgtype.UUID
Collectorid pgtype.UUID `db:"collectorid"`
Activeversion int32 `db:"activeversion"`
Queryid pgtype.UUID `db:"queryid"`
}
type Activequeryrequirement struct {
ID pgtype.UUID
Type Querytype
Activeversion int32
Requiredqueryid pgtype.UUID
ID pgtype.UUID `db:"id"`
Type Querytype `db:"type"`
Activeversion int32 `db:"activeversion"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
}
type Collector struct {
ID pgtype.UUID
Jobid pgtype.UUID
Mincleanversion int32
Mintextversion int32
Latestversion int32
Activeversion int32
ID pgtype.UUID `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
Latestversion int32 `db:"latestversion"`
Activeversion int32 `db:"activeversion"`
}
type Collectorquery struct {
ID pgtype.UUID
Collectorid pgtype.UUID
Name string
Queryid pgtype.UUID
Addedversion int32
Removedversion pgtype.Int4
ID pgtype.UUID `db:"id"`
Collectorid pgtype.UUID `db:"collectorid"`
Name string `db:"name"`
Queryid pgtype.UUID `db:"queryid"`
Addedversion int32 `db:"addedversion"`
Removedversion *int32 `db:"removedversion"`
}
type Collectorquerydependencytree struct {
Collectorid pgtype.UUID
ID pgtype.UUID
Type Querytype
Requiredqueryid pgtype.UUID
Queryversion int32
Collectorid pgtype.UUID `db:"collectorid"`
ID pgtype.UUID `db:"id"`
Type Querytype `db:"type"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
Queryversion int32 `db:"queryversion"`
}
type Fullactivequery struct {
ID pgtype.UUID
Type Querytype
Activeversion int32
Latestversion int32
Config []byte
Requiredids []pgtype.UUID
ID pgtype.UUID `db:"id"`
Type Querytype `db:"type"`
Activeversion int32 `db:"activeversion"`
Latestversion int32 `db:"latestversion"`
Config []byte `db:"config"`
Requiredids []pgtype.UUID `db:"requiredids"`
}
type Query struct {
ID pgtype.UUID
Latestversion int32
Activeversion int32
Type Querytype
ID pgtype.UUID `db:"id"`
Latestversion int32 `db:"latestversion"`
Activeversion int32 `db:"activeversion"`
Type Querytype `db:"type"`
}
type Queryconfig struct {
ID pgtype.UUID
Queryid pgtype.UUID
Config []byte
Addedversion int32
Removedversion pgtype.Int4
ID pgtype.UUID `db:"id"`
Queryid pgtype.UUID `db:"queryid"`
Config []byte `db:"config"`
Addedversion int32 `db:"addedversion"`
Removedversion *int32 `db:"removedversion"`
}
type Querydeprecation struct {
ID pgtype.UUID
Queryid pgtype.UUID
Time pgtype.Timestamp
Removedat pgtype.Timestamp
ID pgtype.UUID `db:"id"`
Queryid pgtype.UUID `db:"queryid"`
Time pgtype.Timestamp `db:"time"`
Removedat pgtype.Timestamp `db:"removedat"`
}
type Requiredquery struct {
ID pgtype.UUID
Queryid pgtype.UUID
Requiredqueryid pgtype.UUID
Addedversion int32
Removedversion pgtype.Int4
ID pgtype.UUID `db:"id"`
Queryid pgtype.UUID `db:"queryid"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
Addedversion int32 `db:"addedversion"`
Removedversion *int32 `db:"removedversion"`
}
type Result struct {
ID pgtype.UUID
Queryid pgtype.UUID
Documentid pgtype.UUID
Value string
Cleanversion int32
Textversion int32
Queryversion int32
ID pgtype.UUID `db:"id"`
Queryid pgtype.UUID `db:"queryid"`
Documentid pgtype.UUID `db:"documentid"`
Value string `db:"value"`
Cleanversion int32 `db:"cleanversion"`
Textversion int32 `db:"textversion"`
Queryversion int32 `db:"queryversion"`
}
internal/database/repository/query.sql.go
+45 -19
View File
@@ -15,6 +15,9 @@ const createQuery = `-- name: CreateQuery :one
INSERT INTO queries (type) VALUES ($1) RETURNING id
`
// CreateQuery
//
// INSERT INTO queries (type) VALUES ($1) RETURNING id
func (q *Queries) CreateQuery(ctx context.Context, type_ Querytype) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, createQuery, type_)
var id pgtype.UUID
@@ -27,12 +30,15 @@ INSERT INTO queryConfigs (queryId, config, addedVersion) VALUES ($1, $2, $3)
`
type CreateQueryConfigParams struct {
Queryid pgtype.UUID
Config []byte
Addedversion int32
Queryid pgtype.UUID `db:"queryid"`
Config []byte `db:"config"`
Addedversion int32 `db:"addedversion"`
}
func (q *Queries) CreateQueryConfig(ctx context.Context, arg CreateQueryConfigParams) error {
// CreateQueryConfig
//
// INSERT INTO queryConfigs (queryId, config, addedVersion) VALUES ($1, $2, $3)
func (q *Queries) CreateQueryConfig(ctx context.Context, arg *CreateQueryConfigParams) error {
_, err := q.db.Exec(ctx, createQueryConfig, arg.Queryid, arg.Config, arg.Addedversion)
return err
}
@@ -42,12 +48,15 @@ INSERT INTO requiredQueries (queryId, requiredQueryId, addedVersion) VALUES ($1,
`
type CreateRequiredQueryParams struct {
Queryid pgtype.UUID
Requiredqueryid pgtype.UUID
Addedversion int32
Queryid pgtype.UUID `db:"queryid"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
Addedversion int32 `db:"addedversion"`
}
func (q *Queries) CreateRequiredQuery(ctx context.Context, arg CreateRequiredQueryParams) error {
// CreateRequiredQuery
//
// INSERT INTO requiredQueries (queryId, requiredQueryId, addedVersion) VALUES ($1, $2, $3)
func (q *Queries) CreateRequiredQuery(ctx context.Context, arg *CreateRequiredQueryParams) error {
_, err := q.db.Exec(ctx, createRequiredQuery, arg.Queryid, arg.Requiredqueryid, arg.Addedversion)
return err
}
@@ -56,6 +65,9 @@ const deprecateQuery = `-- name: DeprecateQuery :exec
INSERT INTO queryDeprecations (queryId) VALUES ($1)
`
// DeprecateQuery
//
// INSERT INTO queryDeprecations (queryId) VALUES ($1)
func (q *Queries) DeprecateQuery(ctx context.Context, queryid pgtype.UUID) error {
_, err := q.db.Exec(ctx, deprecateQuery, queryid)
return err
@@ -65,7 +77,10 @@ const getQuery = `-- name: GetQuery :one
SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries WHERE id = $1
`
func (q *Queries) GetQuery(ctx context.Context, id pgtype.UUID) (Fullactivequery, error) {
// GetQuery
//
// SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries WHERE id = $1
func (q *Queries) GetQuery(ctx context.Context, id pgtype.UUID) (*Fullactivequery, error) {
row := q.db.QueryRow(ctx, getQuery, id)
var i Fullactivequery
err := row.Scan(
@@ -76,7 +91,7 @@ func (q *Queries) GetQuery(ctx context.Context, id pgtype.UUID) (Fullactivequery
&i.Config,
&i.Requiredids,
)
return i, err
return &i, err
}
const getQueryConfig = `-- name: GetQueryConfig :one
@@ -84,20 +99,23 @@ SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 an
`
type GetQueryConfigParams struct {
Queryid pgtype.UUID
Addedversion int32
Queryid pgtype.UUID `db:"queryid"`
Addedversion int32 `db:"addedversion"`
}
type GetQueryConfigRow struct {
ID pgtype.UUID
Config []byte
ID pgtype.UUID `db:"id"`
Config []byte `db:"config"`
}
func (q *Queries) GetQueryConfig(ctx context.Context, arg GetQueryConfigParams) (GetQueryConfigRow, error) {
// GetQueryConfig
//
// SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 and COALESCE(removedVersion, $2 - 1) < $2
func (q *Queries) GetQueryConfig(ctx context.Context, arg *GetQueryConfigParams) (*GetQueryConfigRow, error) {
row := q.db.QueryRow(ctx, getQueryConfig, arg.Queryid, arg.Addedversion)
var i GetQueryConfigRow
err := row.Scan(&i.ID, &i.Config)
return i, err
return &i, err
}
const isQueryDeprecated = `-- name: IsQueryDeprecated :one
@@ -106,6 +124,11 @@ SELECT EXISTS (
)
`
// IsQueryDeprecated
//
// SELECT EXISTS (
// SELECT 1 FROM queryDeprecations where queryId = $1 and removedAt is null LIMIT 1
// )
func (q *Queries) IsQueryDeprecated(ctx context.Context, queryid pgtype.UUID) (bool, error) {
row := q.db.QueryRow(ctx, isQueryDeprecated, queryid)
var exists bool
@@ -117,13 +140,16 @@ const listQueries = `-- name: ListQueries :many
SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries
`
func (q *Queries) ListQueries(ctx context.Context) ([]Fullactivequery, error) {
// ListQueries
//
// SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries
func (q *Queries) ListQueries(ctx context.Context) ([]*Fullactivequery, error) {
rows, err := q.db.Query(ctx, listQueries)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Fullactivequery
items := []*Fullactivequery{}
for rows.Next() {
var i Fullactivequery
if err := rows.Scan(
@@ -136,7 +162,7 @@ func (q *Queries) ListQueries(ctx context.Context) ([]Fullactivequery, error) {
); err != nil {
return nil, err
}
items = append(items, i)
items = append(items, &i)
}
if err := rows.Err(); err != nil {
return nil, err
internal/database/repository/query_test.go
+44 -24
View File
@@ -2,7 +2,6 @@ package repository_test
import (
"context"
"log"
"os"
"path"
"queryorchestration/internal/database"
@@ -35,14 +34,14 @@ func TestQueries(t *testing.T) {
jsonConfig := []byte("{\"path\": \"example_path\"}")
err = queries.CreateRequiredQuery(ctx, repository.CreateRequiredQueryParams{
err = queries.CreateRequiredQuery(ctx, &repository.CreateRequiredQueryParams{
Queryid: jsonQueryID,
Requiredqueryid: contextQueryID,
Addedversion: 1,
})
assert.Nil(t, err)
err = queries.CreateQueryConfig(ctx, repository.CreateQueryConfigParams{
err = queries.CreateQueryConfig(ctx, &repository.CreateQueryConfigParams{
Queryid: jsonQueryID,
Config: jsonConfig,
Addedversion: 1,
@@ -58,34 +57,15 @@ func TestQueries(t *testing.T) {
Latestversion: 1,
Config: jsonConfig,
Requiredids: []pgtype.UUID{contextQueryID},
}, jsonQuery)
}, *jsonQuery)
jsonQueryConfig, err := queries.GetQueryConfig(ctx, repository.GetQueryConfigParams{
jsonQueryConfig, err := queries.GetQueryConfig(ctx, &repository.GetQueryConfigParams{
Queryid: jsonQueryID,
Addedversion: jsonQuery.Activeversion,
})
assert.Nil(t, err)
assert.Equal(t, jsonConfig, jsonQueryConfig.Config)
qs, err := queries.ListQueries(ctx)
assert.Nil(t, err)
assert.Len(t, qs, 2)
log.Print(qs)
assert.ElementsMatch(t, []repository.Fullactivequery{
{ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Latestversion: 1,
Config: jsonConfig,
Requiredids: []pgtype.UUID{contextQueryID}},
{ID: contextQueryID,
Type: repository.QuerytypeContextFull,
Activeversion: 1,
Latestversion: 1,
Config: nil,
Requiredids: []pgtype.UUID{{}}},
}, qs)
isDeprecated, err := queries.IsQueryDeprecated(ctx, jsonQueryID)
assert.Nil(t, err)
assert.False(t, isDeprecated)
@@ -97,3 +77,43 @@ func TestQueries(t *testing.T) {
assert.Nil(t, err)
assert.True(t, isDeprecated)
}
func TestQueriesList(t *testing.T) {
ctx := context.Background()
db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Migrations: &database.MigrationConfig{
BasePath: "../../..",
}})
defer cleanup()
queries := repository.New(db.Pool)
contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
assert.Nil(t, err)
jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.Nil(t, err)
qs, err := queries.ListQueries(ctx)
assert.Nil(t, err)
assert.Len(t, qs, 2)
assert.ElementsMatch(t, []*repository.Fullactivequery{
{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Latestversion: 1,
Config: nil,
Requiredids: []pgtype.UUID{{}},
},
{
ID: contextQueryID,
Type: repository.QuerytypeContextFull,
Activeversion: 1,
Latestversion: 1,
Config: nil,
Requiredids: []pgtype.UUID{{}},
},
}, qs)
}
internal/database/repository/result.sql.go
+31 -22
View File
@@ -16,24 +16,27 @@ SELECT id, queryId, value FROM results where id = ANY($1)
`
type ListResultValuesByIDRow struct {
ID pgtype.UUID
Queryid pgtype.UUID
Value string
ID pgtype.UUID `db:"id"`
Queryid pgtype.UUID `db:"queryid"`
Value string `db:"value"`
}
func (q *Queries) ListResultValuesByID(ctx context.Context, id []pgtype.UUID) ([]ListResultValuesByIDRow, error) {
// ListResultValuesByID
//
// SELECT id, queryId, value FROM results where id = ANY($1)
func (q *Queries) ListResultValuesByID(ctx context.Context, id []pgtype.UUID) ([]*ListResultValuesByIDRow, error) {
rows, err := q.db.Query(ctx, listResultValuesByID, id)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListResultValuesByIDRow
items := []*ListResultValuesByIDRow{}
for rows.Next() {
var i ListResultValuesByIDRow
if err := rows.Scan(&i.ID, &i.Queryid, &i.Value); err != nil {
return nil, err
}
items = append(items, i)
items = append(items, &i)
}
if err := rows.Err(); err != nil {
return nil, err
@@ -46,30 +49,33 @@ SELECT id, queryId, queryVersion FROM results where documentId = $1 and cleanVer
`
type ListResultsByDocumentIDParams struct {
Documentid pgtype.UUID
Cleanversion int32
Textversion int32
Documentid pgtype.UUID `db:"documentid"`
Cleanversion int32 `db:"cleanversion"`
Textversion int32 `db:"textversion"`
}
type ListResultsByDocumentIDRow struct {
ID pgtype.UUID
Queryid pgtype.UUID
Queryversion int32
ID pgtype.UUID `db:"id"`
Queryid pgtype.UUID `db:"queryid"`
Queryversion int32 `db:"queryversion"`
}
func (q *Queries) ListResultsByDocumentID(ctx context.Context, arg ListResultsByDocumentIDParams) ([]ListResultsByDocumentIDRow, error) {
// ListResultsByDocumentID
//
// SELECT id, queryId, queryVersion FROM results where documentId = $1 and cleanVersion >= $2 and textVersion >= $3
func (q *Queries) ListResultsByDocumentID(ctx context.Context, arg *ListResultsByDocumentIDParams) ([]*ListResultsByDocumentIDRow, error) {
rows, err := q.db.Query(ctx, listResultsByDocumentID, arg.Documentid, arg.Cleanversion, arg.Textversion)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListResultsByDocumentIDRow
items := []*ListResultsByDocumentIDRow{}
for rows.Next() {
var i ListResultsByDocumentIDRow
if err := rows.Scan(&i.ID, &i.Queryid, &i.Queryversion); err != nil {
return nil, err
}
items = append(items, i)
items = append(items, &i)
}
if err := rows.Err(); err != nil {
return nil, err
@@ -82,15 +88,18 @@ INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, quer
`
type SetResultParams struct {
Queryid pgtype.UUID
Documentid pgtype.UUID
Value string
Cleanversion int32
Textversion int32
Queryversion int32
Queryid pgtype.UUID `db:"queryid"`
Documentid pgtype.UUID `db:"documentid"`
Value string `db:"value"`
Cleanversion int32 `db:"cleanversion"`
Textversion int32 `db:"textversion"`
Queryversion int32 `db:"queryversion"`
}
func (q *Queries) SetResult(ctx context.Context, arg SetResultParams) (pgtype.UUID, error) {
// SetResult
//
// INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id
func (q *Queries) SetResult(ctx context.Context, arg *SetResultParams) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, setResult,
arg.Queryid,
arg.Documentid,
internal/database/repository/result_test.go
+4 -4
View File
@@ -37,7 +37,7 @@ func TestResults(t *testing.T) {
textVersion := int32(1)
jsonResultValue := "example_value"
jsonResultID, err := queries.SetResult(ctx, repository.SetResultParams{
jsonResultID, err := queries.SetResult(ctx, &repository.SetResultParams{
Queryid: jsonQueryID,
Documentid: documentID,
Value: jsonResultValue,
@@ -48,14 +48,14 @@ func TestResults(t *testing.T) {
assert.Nil(t, err)
assert.True(t, jsonResultID.Valid)
resultsByDoc, err := queries.ListResultsByDocumentID(ctx, repository.ListResultsByDocumentIDParams{
resultsByDoc, err := queries.ListResultsByDocumentID(ctx, &repository.ListResultsByDocumentIDParams{
Documentid: documentID,
Cleanversion: cleanVersion,
Textversion: textVersion,
})
assert.Nil(t, err)
assert.Len(t, resultsByDoc, 1)
assert.ElementsMatch(t, []repository.ListResultsByDocumentIDRow{
assert.ElementsMatch(t, []*repository.ListResultsByDocumentIDRow{
{
ID: jsonResultID,
Queryid: jsonQueryID,
@@ -66,7 +66,7 @@ func TestResults(t *testing.T) {
results, err := queries.ListResultValuesByID(ctx, []pgtype.UUID{jsonResultID})
assert.Nil(t, err)
assert.Len(t, results, 1)
assert.ElementsMatch(t, []repository.ListResultValuesByIDRow{
assert.ElementsMatch(t, []*repository.ListResultValuesByIDRow{
{
ID: jsonResultID,
Queryid: jsonQueryID,
internal/document/service.go
+2 -2
View File
@@ -56,7 +56,7 @@ func (s *Service) Sync(ctx context.Context, doc *Document) error {
func (s *Service) getResults(ctx context.Context, id uuid.UUID, coll *collector.Collector) ([]*result.Result, error) {
docID := database.MustToDBUUID(id)
results, err := s.db.Queries.ListResultsByDocumentID(ctx, repository.ListResultsByDocumentIDParams{
results, err := s.db.Queries.ListResultsByDocumentID(ctx, &repository.ListResultsByDocumentIDParams{
Documentid: docID,
Textversion: coll.MinTextVersion,
Cleanversion: coll.MinCleanVersion,
@@ -67,7 +67,7 @@ func (s *Service) getResults(ctx context.Context, id uuid.UUID, coll *collector.
cleanResults := make([]*result.Result, len(results))
for index, dbResult := range results {
cleanResults[index] = result.Parse(&dbResult)
cleanResults[index] = result.Parse(dbResult)
}
return cleanResults, nil
internal/document/sync_test.go
+7 -4
View File
@@ -37,6 +37,7 @@ func TestSyncIsSynced(t *testing.T) {
dbJobID := database.MustToDBUUID(doc.JobID)
minCleanVersion := int32(1)
minTextVersion := int32(1)
qV := int32(1)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
WillReturnRows(
@@ -46,12 +47,12 @@ func TestSyncIsSynced(t *testing.T) {
pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}).
AddRow(pgtype.UUID{}, pgtype.UUID{}, int32(1)),
AddRow(pgtype.UUID{}, pgtype.UUID{}, qV),
)
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorId).
WillReturnRows(
pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
AddRow(dbCollectorId, pgtype.UUID{}, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, pgtype.Int4{Int32: int32(1), Valid: true}, []pgtype.UUID{}),
AddRow(dbCollectorId, pgtype.UUID{}, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, &qV, []pgtype.UUID{}),
)
docSvc := document.New(db)
@@ -133,10 +134,11 @@ func TestSyncDBFail(t *testing.T) {
pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}),
)
dbErr = "database failure"
qV := int32(1)
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorId).
WillReturnRows(
pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
AddRow(dbCollectorId, dbQueryID, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, pgtype.Int4{Int32: int32(1), Valid: true}, []pgtype.UUID{}),
AddRow(dbCollectorId, dbQueryID, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, &qV, []pgtype.UUID{}),
)
pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{}).
WillReturnError(errors.New(dbErr))
@@ -180,10 +182,11 @@ func TestSync(t *testing.T) {
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}),
)
qV := int32(1)
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorId).
WillReturnRows(
pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
AddRow(dbCollectorId, dbQueryID, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, pgtype.Int4{Int32: int32(1), Valid: true}, []pgtype.UUID{}),
AddRow(dbCollectorId, dbQueryID, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, &qV, []pgtype.UUID{}),
)
pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{}).
WillReturnRows(
internal/jsonExtractor/service.go
+1 -1
View File
@@ -34,7 +34,7 @@ func (e Extractor) Process(ctx context.Context, query *queryprocessor.Query, val
return "", err
}
byteConfig, err := e.db.Queries.GetQueryConfig(ctx, repository.GetQueryConfigParams{
byteConfig, err := e.db.Queries.GetQueryConfig(ctx, &repository.GetQueryConfigParams{
Queryid: database.MustToDBUUID(query.ID),
Addedversion: query.Version,
})
internal/query/create.go
+2 -2
View File
@@ -54,7 +54,7 @@ func (s *Service) submitCreate(ctx context.Context, entity *queryprocessor.Creat
}
for _, reqQuery := range query.RequiredQueryIDs {
err = qtx.CreateRequiredQuery(ctx, repository.CreateRequiredQueryParams{
err = qtx.CreateRequiredQuery(ctx, &repository.CreateRequiredQueryParams{
Queryid: dbID,
Requiredqueryid: reqQuery,
Addedversion: 1,
@@ -65,7 +65,7 @@ func (s *Service) submitCreate(ctx context.Context, entity *queryprocessor.Creat
}
if query.Config != nil && string(query.Config) != "" {
err = qtx.CreateQueryConfig(ctx, repository.CreateQueryConfigParams{
err = qtx.CreateQueryConfig(ctx, &repository.CreateQueryConfigParams{
Queryid: dbID,
Config: query.Config,
Addedversion: 1,
internal/query/get.go
+15 -1
View File
@@ -24,7 +24,7 @@ func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Query, error) {
return nil, err
}
return ParseFullActiveQuery(&query)
return ParseFullActiveQuery(query)
}
func ParseFullActiveQuery(q *repository.Fullactivequery) (*Query, error) {
@@ -47,3 +47,17 @@ func ParseFullActiveQuery(q *repository.Fullactivequery) (*Query, error) {
Config: string(q.Config),
}, nil
}
func ParseFullActiveQueryArray(q []*repository.Fullactivequery) ([]*Query, error) {
queries := make([]*Query, len(q))
for index, query := range q {
q, err := ParseFullActiveQuery(query)
if err != nil {
return nil, err
}
queries[index] = q
}
return queries, nil
}
internal/query/get_test.go
+26
View File
@@ -96,3 +96,29 @@ func TestFullActiveQueryWithNullUUID(t *testing.T) {
RequiredQueryIDs: []uuid.UUID{},
}, *out)
}
func TestFullActiveQueryArray(t *testing.T) {
dbQueries := []*repository.Fullactivequery{
{
ID: database.MustToDBUUID(uuid.New()),
Type: repository.QuerytypeContextFull,
Activeversion: int32(1),
Latestversion: int32(2),
Config: []byte(""),
Requiredids: []pgtype.UUID{},
},
}
out, err := query.ParseFullActiveQueryArray(dbQueries)
assert.Nil(t, err)
assert.EqualExportedValues(t, []*query.Query{
{
ID: database.MustToUUID(dbQueries[0].ID),
Type: queryprocessor.TypeContextFull,
ActiveVersion: int32(1),
LatestVersion: int32(2),
Config: "",
RequiredQueryIDs: []uuid.UUID{},
},
}, out)
}
internal/query/list.go
+3 -12
View File
@@ -2,29 +2,20 @@ package query
import (
"context"
queryprocessor "queryorchestration/internal/queryProcessor"
)
type ListFilters struct {
Types []queryprocessor.Type
}
func (s *Service) List(ctx context.Context, filters ListFilters) ([]*Query, error) {
// TODO - use filters
dbQueries, err := s.db.Queries.ListQueries(ctx)
if err != nil {
return nil, err
}
queries := make([]*Query, len(dbQueries))
for index, query := range dbQueries {
q, err := ParseFullActiveQuery(&query)
if err != nil {
return nil, err
}
queries[index] = q
queries, err := ParseFullActiveQueryArray(dbQueries)
if err != nil {
return nil, err
}
return queries, nil
internal/query/list_test.go
+41
View File
@@ -53,3 +53,44 @@ func TestList(t *testing.T) {
assert.EqualExportedValues(t, []*query.Query{q}, resList)
}
func TestListFilterType(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := query.New(db)
config := "{\"path\":\"example_path\"}"
q := &query.Query{
ID: uuid.New(),
Type: queryprocessor.TypeJsonExtractor,
ActiveVersion: int32(1),
LatestVersion: int32(1),
RequiredQueryIDs: []uuid.UUID{
uuid.New(),
},
Config: config,
}
dbReqIDs := database.MustToDBUUIDArray(q.RequiredQueryIDs)
filters := query.ListFilters{}
pool.ExpectQuery("name: ListQueries :many").WithArgs().WillReturnRows(
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
AddRow(database.MustToDBUUID(q.ID), repository.QuerytypeJsonExtractor, q.ActiveVersion, q.LatestVersion, []byte(config), dbReqIDs),
)
resList, err := svc.List(ctx, filters)
assert.Nil(t, err)
assert.EqualExportedValues(t, []*query.Query{q}, resList)
}
internal/queryProcessor/parse.go
+15 -1
View File
@@ -40,6 +40,20 @@ func ToDBQueryType(t Type) (repository.Querytype, error) {
return dbType, nil
}
func ToDBQueryTypeArray(t []Type) ([]repository.Querytype, error) {
arr := make([]repository.Querytype, len(t))
for index, value := range t {
v, err := ToDBQueryType(value)
if err != nil {
return nil, err
}
arr[index] = v
}
return arr, nil
}
func ToDBNullQueryType(t Type) (repository.NullQuerytype, error) {
dbType, err := ToDBQueryType(t)
if err != nil {
@@ -59,7 +73,7 @@ func ParseDBCollectorQuery(q *repository.GetCollectorQueriesRow) (*Query, error)
return &Query{
ID: database.MustToUUID(q.Queryid),
Version: q.Queryversion.Int32,
Version: *q.Queryversion,
Type: qType,
RequiredQueryIDs: reqQueryIDs,
}, nil
internal/queryProcessor/parse_test.go
+21 -1
View File
@@ -11,12 +11,13 @@ import (
)
func TestParseDBCollectorQuery(t *testing.T) {
qV := int32(0)
dbResult := repository.GetCollectorQueriesRow{
Collectorid: pgtype.UUID{},
Queryid: pgtype.UUID{},
Requiredids: []pgtype.UUID{},
Type: repository.NullQuerytype{Valid: true, Querytype: repository.QuerytypeJsonExtractor},
Queryversion: pgtype.Int4{},
Queryversion: &qV,
}
value, err := queryprocessor.ParseDBCollectorQuery(&dbResult)
assert.Nil(t, err)
@@ -73,6 +74,25 @@ func TestToDBQueryType(t *testing.T) {
assert.Equal(t, repository.Querytype(repository.QuerytypeContextFull), value)
}
func TestToDBQueryTypeArray(t *testing.T) {
inArr := []queryprocessor.Type{
queryprocessor.TypeJsonExtractor,
queryprocessor.TypeContextFull,
}
value, err := queryprocessor.ToDBQueryTypeArray(inArr)
assert.Nil(t, err)
assert.Equal(t, []repository.Querytype{
repository.QuerytypeJsonExtractor,
repository.QuerytypeContextFull,
}, value)
inArr = []queryprocessor.Type{
queryprocessor.Type(-1),
}
_, err = queryprocessor.ToDBQueryTypeArray(inArr)
assert.EqualError(t, err, "invalid database query type")
}
func TestToDBNullQueryType(t *testing.T) {
dbQueryType := queryprocessor.Type(queryprocessor.TypeJsonExtractor)
value, err := queryprocessor.ToDBNullQueryType(dbQueryType)
internal/queryQueue/create.go
+1 -1
View File
@@ -40,7 +40,7 @@ func (c *Queue) getCollectorQueries(ctx context.Context) error {
cleanQueries := make([]*queryprocessor.Query, len(queries))
for index, dbQuery := range queries {
cleanQuery, err := queryprocessor.ParseDBCollectorQuery(&dbQuery)
cleanQuery, err := queryprocessor.ParseDBCollectorQuery(dbQuery)
if err != nil {
return err
}
internal/queryQueue/create_test.go
+1 -2
View File
@@ -10,7 +10,6 @@ import (
"testing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
)
@@ -71,7 +70,7 @@ func TestGetCollectorQueries(t *testing.T) {
ty, err := queryprocessor.ToDBNullQueryType(q.Type)
assert.Nil(t, err)
rows = rows.
AddRow(dbCollectorID, dbID, ty, pgtype.Int4{Int32: int32(q.Version), Valid: true}, dbReqIDs)
AddRow(dbCollectorID, dbID, ty, &q.Version, dbReqIDs)
}
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
internal/queryQueue/execute.go
+1 -1
View File
@@ -52,7 +52,7 @@ func (q *Queue) executeQuery(ctx context.Context, qu *queryprocessor.Query) erro
cleanValues := make([]result.Value, len(values))
for index, r := range values {
cleanValue, err := q.getResultValue(&r)
cleanValue, err := q.getResultValue(r)
if err != nil {
return err
}
internal/queryQueue/execute_test.go
+1 -1
View File
@@ -74,7 +74,7 @@ func TestExecute(t *testing.T) {
ty, err := queryprocessor.ToDBNullQueryType(q.Type)
assert.Nil(t, err)
rows = rows.
AddRow(dbCollectorID, dbID, ty, pgtype.Int4{Int32: int32(q.Version), Valid: true}, dbReqIDs)
AddRow(dbCollectorID, dbID, ty, &q.Version, dbReqIDs)
}
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
internal/queryQueue/service_test.go
+1 -2
View File
@@ -12,7 +12,6 @@ import (
"testing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
)
@@ -74,7 +73,7 @@ func TestService(t *testing.T) {
ty, err := queryprocessor.ToDBNullQueryType(q.Type)
assert.Nil(t, err)
rows = rows.
AddRow(dbCollectorID, dbID, ty, pgtype.Int4{Int32: int32(q.Version), Valid: true}, dbReqIDs)
AddRow(dbCollectorID, dbID, ty, &q.Version, dbReqIDs)
}
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
internal/result/service.go
+1 -1
View File
@@ -28,7 +28,7 @@ type ResultStore struct {
}
func Store(ctx context.Context, dbQueries *repository.Queries, res *ResultStore) (uuid.UUID, error) {
dbId, err := dbQueries.SetResult(ctx, repository.SetResultParams{
dbId, err := dbQueries.SetResult(ctx, &repository.SetResultParams{
Queryid: database.MustToDBUUID(res.QueryID),
Documentid: database.MustToDBUUID(res.DocumentID),
Value: res.Value,
internal/server/server_test.go
+1 -3
View File
@@ -14,9 +14,7 @@ import (
func TestNew(t *testing.T) {
ctx := context.Background()
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
NoPool: true,
})
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{})
defer cleanup()
newCfg := &server.NewConfig{
internal/test/container.go
+6 -1
View File
@@ -43,12 +43,17 @@ func createContainer(t *testing.T, ctx context.Context, cfg *containerConfig) (t
Entrypoint: []string{fmt.Sprintf("./%s", cfg.ServiceName)},
}
if cfg.ExposedPorts != nil {
if len(cfg.ExposedPorts) > 0 {
ports := make([]string, len(cfg.ExposedPorts))
for index, port := range cfg.ExposedPorts {
ports[index] = port.Port()
}
req.ExposedPorts = ports
req.WaitingFor = wait.ForAll(
wait.ForExposedPort(),
wait.ForListeningPort(cfg.ExposedPorts[0]),
wait.ForLog(cfg.WaitForMsg),
)
}
container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
internal/test/database.go
+7 -7
View File
@@ -29,7 +29,6 @@ type Database struct {
type CreateDatabaseConfig struct {
Network *testcontainers.DockerNetwork
Migrations *database.MigrationConfig
NoPool bool
}
func CreateDB(t *testing.T, ctx context.Context, cfg *CreateDatabaseConfig) (*Database, func()) {
@@ -52,8 +51,11 @@ func CreateDB(t *testing.T, ctx context.Context, cfg *CreateDatabaseConfig) (*Da
"POSTGRES_PASSWORD": pass,
},
ExposedPorts: []string{port.Port()},
// WaitingFor: wait.ForLog("database system is ready to accept connections"),
WaitingFor: wait.ForExposedPort(),
WaitingFor: wait.ForAll(
wait.ForExposedPort(),
wait.ForListeningPort(port),
wait.ForLog("database system is ready to accept connections"),
),
}
if cfg.Network != nil {
@@ -87,17 +89,15 @@ func CreateDB(t *testing.T, ctx context.Context, cfg *CreateDatabaseConfig) (*Da
t.Setenv("DB_NAME", name)
t.Setenv("DB_NOSSL", "1")
var pool *pgxpool.Pool
if cfg.Migrations != nil {
database.RunMigrations(ctx, cfg.Migrations)
}
var pool *pgxpool.Pool
if !cfg.NoPool {
pool = database.GetDBPool(ctx)
err = pool.Ping(ctx)
if err != nil {
t.Fatal("Unable to ping database")
t.Fatalf("Unable to ping database: %s", err)
}
}
internal/test/database_test.go
+3 -3
View File
@@ -14,10 +14,9 @@ import (
func TestCreateDB(t *testing.T) {
ctx := context.Background()
dbcfg, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
NoPool: true,
})
dbcfg, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{})
assert.NotNil(t, dbcfg)
assert.Nil(t, dbcfg.Pool)
assert.NotNil(t, cleanup)
cleanup()
@@ -32,6 +31,7 @@ func TestCreateDBWithMigrations(t *testing.T) {
},
})
assert.NotNil(t, dbcfg)
assert.NotNil(t, dbcfg.Pool)
assert.NotNil(t, cleanup)
cleanup()
internal/test/queue.go
+5 -1
View File
@@ -61,7 +61,11 @@ func CreateQueue(t *testing.T, ctx context.Context, cfg *CreateQueueConfig) (*Qu
"SQS_ENDPOINT_STRATEGY": "path",
},
ExposedPorts: []string{port.Port()},
WaitingFor: wait.ForListeningPort(port),
WaitingFor: wait.ForAll(
// wait.ForExposedPort(),
wait.ForListeningPort(port),
wait.ForLog("Ready."),
),
}
if cfg.Network != nil {
internal/test/queuecontainer_test.go
+6
View File
@@ -2,6 +2,9 @@ package test
import (
"context"
"os"
"path"
"queryorchestration/internal/database"
"testing"
"github.com/stretchr/testify/assert"
@@ -13,6 +16,9 @@ func TestCreateQueueContainer(t *testing.T) {
ncfg, ncleanup := CreateNetwork(t, ctx)
defer ncleanup()
dbcfg, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Migrations: &database.MigrationConfig{
BasePath: path.Join(os.Getenv("PWD"), "../.."),
},
Network: ncfg,
})
defer dbcleanup()
scripts/database.yml
-2
View File
@@ -9,8 +9,6 @@ vars:
tasks:
generate:
cmds:
- task compose:up:bg
- sleep 2
- task db:mig:run
- sqlc generate --file sqlc.yml
lint:
serviceInterfaces/main.proto
+1 -3
View File
@@ -40,9 +40,7 @@ message Query {
repeated string required_queries = 6;
}
message QueryFilter {
repeated QueryType types = 1;
}
message QueryFilter {}
message QueryCreate {
QueryType type = 1;
sqlc.yml
+7
View File
@@ -21,6 +21,13 @@ sql:
package: "repository"
out: "internal/database/repository"
sql_package: "pgx/v5"
emit_pointers_for_null_types: true
emit_empty_slices: true
emit_db_tags: true
emit_result_struct_pointers: true
emit_params_struct_pointers: true
emit_enum_valid_method: true
emit_sql_as_comment: true
rules:
- name: no-pg
message: "invalid engine: not postgresql"
test/queryService/listqueries_test.go
+12 -11
View File
@@ -17,23 +17,24 @@ func TestQueryServiceList(t *testing.T) {
client := serviceinterfaces.NewQueryServiceClient(conn)
idRes, err := client.Create(ctx, &serviceinterfaces.QueryCreate{
contextRes, err := client.Create(ctx, &serviceinterfaces.QueryCreate{
Type: serviceinterfaces.QueryType_QUERY_TYPE_CONTEXT_FULL,
Config: nil,
RequiredQueries: []string{},
})
assert.Nil(t, err)
id := idRes.GetId()
assert.NotEmpty(t, id)
config := "\"path\":\".key\""
_, err = client.Create(ctx, &serviceinterfaces.QueryCreate{
Type: serviceinterfaces.QueryType_QUERY_TYPE_JSON_EXTRACTOR,
Config: &config,
RequiredQueries: []string{
contextRes.Id,
},
})
assert.Nil(t, err)
queriesRes, err := client.List(ctx, &serviceinterfaces.QueryFilter{})
assert.Nil(t, err)
assert.Len(t, queriesRes.Queries, 1)
assert.Equal(t, id, queriesRes.Queries[0].Id)
res, err := client.Update(ctx, &serviceinterfaces.QueryUpdate{
Id: id,
})
assert.Nil(t, err)
assert.NotNil(t, res)
assert.Len(t, queriesRes.Queries, 2)
}