jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/remove-query (pull request #201)

Browse Source 
remove query from codebase part 1

* remove query

* fix localstack run
This commit is contained in:
Jay Brown
2026-01-14 17:59:04 +00:00
parent ebf47c6013
commit 0ddae4f91e
167 changed files with 1059 additions and 20641 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/test/api.go
+1 -1
View File
@@ -53,11 +53,11 @@ func CreateAPI(t testing.TB, ctx context.Context, cfg ContainerConfigProvider, n
}, cleanup
}
// QueryAPI configuration. Query processing has been removed - only client sync remains.
var QueryAPI = API{
Name: QueryAPIName,
DownstreamQueues: []RunnerName{
ClientSyncRunnerName,
QueryVersionSyncRunnerName,
},
}
internal/test/config.go
+3 -2
View File
@@ -121,9 +121,10 @@ func WaitForAPIRoutes(uri string, timeout time.Duration) error {
client := &http.Client{Timeout: 2 * time.Second}
deadline := time.Now().Add(timeout)
// Check /query endpoint - should return 200 with empty array when routes are ready
// Check /clients endpoint - should return a response when routes are ready
// A "404 page not found" means routes aren't registered yet
routeURL := fmt.Sprintf("%s/query", uri)
// Note: /query endpoint was removed as part of query functionality removal
routeURL := fmt.Sprintf("%s/clients", uri)
slog.Info("Waiting for API routes to be registered", "url", routeURL, "timeout", timeout)
var lastErr error
internal/test/config_test.go
+3 -3
View File
@@ -152,7 +152,7 @@ func TestWaitForExternalReadiness(t *testing.T) {
func TestWaitForAPIRoutes(t *testing.T) {
t.Run("success when routes return 200", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/query" {
if r.URL.Path == "/clients" {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("[]"))
return
@@ -168,7 +168,7 @@ func TestWaitForAPIRoutes(t *testing.T) {
t.Run("success after routes become available", func(t *testing.T) {
attempts := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/query" {
if r.URL.Path == "/clients" {
attempts++
if attempts < 3 {
// Simulate "404 page not found" before routes are registered
@@ -192,7 +192,7 @@ func TestWaitForAPIRoutes(t *testing.T) {
t.Run("success when routes return 401", func(t *testing.T) {
// 401 means the route exists but requires auth - route is registered
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/query" {
if r.URL.Path == "/clients" {
w.WriteHeader(http.StatusUnauthorized)
return
}
internal/test/queryAPI/service.go
+3 -36
View File
@@ -14,12 +14,13 @@ import (
queryapi "queryorchestration/pkg/queryAPI"
"github.com/google/uuid"
"github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Note: Query-related helper functions have been removed.
// See remove_query_plan.md for details.
type File struct {
ClientID queryapi.ClientID
Filename string
@@ -69,22 +70,6 @@ func CreateClientWithSync(t testing.TB, client queryapi.ClientWithResponsesInter
return clientCreateRes.JSON201.Id
}
func SetQueryForClient(t testing.TB, client queryapi.ClientWithResponsesInterface, clientId queryapi.ClientID, queryId uuid.UUID) {
t.Helper()
newActiveVersion := int32(1)
_, err := client.SetCollectorByClientIdWithResponse(t.Context(), clientId, queryapi.CollectorSet{
ActiveVersion: &newActiveVersion,
Fields: &[]queryapi.CollectorField{
{
Name: "JSON_QUERY",
QueryId: queryId,
},
},
})
require.NoError(t, err)
}
func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.ClientWithResponsesInterface, id string, status queryapi.ClientStatus) {
t.Helper()
@@ -125,21 +110,3 @@ func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.Cli
}
}
}
func CreateDependentQueries(t testing.TB, client *queryapi.ClientWithResponses) (uuid.UUID, uuid.UUID) {
contextQueryRes, err := client.CreateQueryWithResponse(t.Context(), queryapi.QueryCreate{
Type: queryapi.CONTEXTFULL,
})
require.NoError(t, err)
test.AssertStatus(t, http.StatusCreated, contextQueryRes.HTTPResponse)
jcfg := `{"path":"keyone"}`
jsonQueryRes, err := client.CreateQueryWithResponse(t.Context(), queryapi.QueryCreate{
Type: queryapi.JSONEXTRACTOR,
Config: &jcfg,
RequiredQueries: &[]types.UUID{contextQueryRes.JSON201.Id},
})
require.NoError(t, err)
test.AssertStatus(t, http.StatusCreated, jsonQueryRes.HTTPResponse)
return contextQueryRes.JSON201.Id, jsonQueryRes.JSON201.Id
}
internal/test/queue.go
-53
View File
@@ -3,7 +3,6 @@ package test
import (
"context"
"fmt"
"log"
"log/slog"
"regexp"
"testing"
@@ -112,55 +111,3 @@ func AssertMessageAttr(t testing.TB, cfg serviceconfig.ConfigProvider, url strin
assert.NotNil(t, message.MessageAttributes[name])
assert.Regexp(t, value, *(message.MessageAttributes[name]).StringValue)
}
func AssertMessageBodies(t testing.TB, cfg serviceconfig.ConfigProvider, url string, bodies []*regexp.Regexp) {
t.Helper()
timeout := time.After(30 * time.Second)
tick := time.NewTicker(2 * time.Second)
defer tick.Stop()
for {
select {
case <-timeout:
t.Fatal("assert timeout")
case <-tick.C:
slog.Info("receiving from queue")
result, err := cfg.ReceiveFromQueue(t.Context(), &queue.ReceiveParams{
QueueURL: url,
VisibilityTimeout: 1,
WaitTimeSeconds: 1,
})
if err != nil {
continue
} else if len(result.Messages) < 1 {
continue
}
found := true
for _, m := range result.Messages {
foundBody := false
for _, b := range bodies {
log.Print(*m.Body)
if b.MatchString(*m.Body) {
foundBody = true
break
}
}
if !foundBody {
found = false
break
}
}
if !found {
continue
}
return
case <-t.Context().Done():
t.Fatal(t.Context().Err())
}
}
}
internal/test/runner.go
+15 -46
View File
@@ -9,38 +9,29 @@ import (
"queryorchestration/internal/serviceconfig/queue/documentinit"
"queryorchestration/internal/serviceconfig/queue/documentsync"
"queryorchestration/internal/serviceconfig/queue/documenttext"
"queryorchestration/internal/serviceconfig/queue/query"
"queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/queue/queryversionsync"
"queryorchestration/internal/serviceconfig/queue/storeevent"
)
type RunnerName string
const (
StoreEventRunnerName RunnerName = "storeEventRunner"
DocInitRunnerName RunnerName = "docInitRunner"
DocSyncRunnerName RunnerName = "docSyncRunner"
DocCleanRunnerName RunnerName = "docCleanRunner"
DocTextRunnerName RunnerName = "docTextRunner"
QuerySyncRunnerName RunnerName = "querySyncRunner"
QueryRunnerName RunnerName = "queryRunner"
ClientSyncRunnerName RunnerName = "clientSyncRunner"
QueryVersionSyncRunnerName RunnerName = "queryVersionSyncRunner"
StoreEventRunnerName RunnerName = "storeEventRunner"
DocInitRunnerName RunnerName = "docInitRunner"
DocSyncRunnerName RunnerName = "docSyncRunner"
DocCleanRunnerName RunnerName = "docCleanRunner"
DocTextRunnerName RunnerName = "docTextRunner"
ClientSyncRunnerName RunnerName = "clientSyncRunner"
)
type RunnerEnv string
const (
StoreEventRunnerEnv RunnerEnv = storeevent.EnvName
DocInitRunnerEnv RunnerEnv = documentinit.EnvName
DocSyncRunnerEnv RunnerEnv = documentsync.EnvName
DocCleanRunnerEnv RunnerEnv = documentclean.EnvName
DocTextRunnerEnv RunnerEnv = documenttext.EnvName
QuerySyncRunnerEnv RunnerEnv = querysync.EnvName
QueryRunnerEnv RunnerEnv = query.EnvName
ClientSyncRunnerEnv RunnerEnv = clientsync.EnvName
QueryVersionSyncRunnerEnv RunnerEnv = queryversionsync.EnvName
StoreEventRunnerEnv RunnerEnv = storeevent.EnvName
DocInitRunnerEnv RunnerEnv = documentinit.EnvName
DocSyncRunnerEnv RunnerEnv = documentsync.EnvName
DocCleanRunnerEnv RunnerEnv = documentclean.EnvName
DocTextRunnerEnv RunnerEnv = documenttext.EnvName
ClientSyncRunnerEnv RunnerEnv = clientsync.EnvName
)
type Runner struct {
@@ -60,14 +51,8 @@ func GetRunnerEnvFromName(name RunnerName) RunnerEnv {
return DocCleanRunnerEnv
case DocTextRunnerName:
return DocTextRunnerEnv
case QuerySyncRunnerName:
return QuerySyncRunnerEnv
case QueryRunnerName:
return QueryRunnerEnv
case ClientSyncRunnerName:
return ClientSyncRunnerEnv
case QueryVersionSyncRunnerName:
return QueryVersionSyncRunnerEnv
}
return DocInitRunnerEnv
}
@@ -117,19 +102,11 @@ var DocCleanRunner = Runner{
DownstreamQueues: []RunnerName{DocTextRunnerName},
}
// DocTextRunner is the terminal runner in the main document processing pipeline.
// Query processing has been removed - pipeline ends after text extraction.
var DocTextRunner = Runner{
Name: DocTextRunnerName,
DownstreamQueues: []RunnerName{QuerySyncRunnerName},
}
var QuerySyncRunner = Runner{
Name: QuerySyncRunnerName,
DownstreamQueues: []RunnerName{QueryRunnerName},
}
var QueryRunner = Runner{
Name: QueryRunnerName,
DownstreamQueues: []RunnerName{QueryRunnerName},
DownstreamQueues: []RunnerName{},
}
var ClientSyncRunner = Runner{
@@ -137,19 +114,11 @@ var ClientSyncRunner = Runner{
DownstreamQueues: []RunnerName{DocSyncRunnerName},
}
var QueryVersionSyncRunner = Runner{
Name: QueryVersionSyncRunnerName,
DownstreamQueues: []RunnerName{ClientSyncRunnerName},
}
var runners = []Runner{
StoreEventRunner,
DocInitRunner,
DocSyncRunner,
DocCleanRunner,
DocTextRunner,
QuerySyncRunner,
QueryRunner,
ClientSyncRunner,
QueryVersionSyncRunner,
}
internal/test/runner_test.go
+2 -4
View File
@@ -6,15 +6,13 @@ import (
"github.com/stretchr/testify/assert"
)
// TestGetRunnerEnvFromName tests the runner environment name lookup.
// Note: Query runner references have been removed. See remove_query_plan.md for details.
func TestGetRunnerEnvFromName(t *testing.T) {
assert.Equal(t, StoreEventRunnerEnv, GetRunnerEnvFromName(StoreEventRunnerName))
assert.Equal(t, DocInitRunnerEnv, GetRunnerEnvFromName(DocInitRunnerName))
assert.Equal(t, DocSyncRunnerEnv, GetRunnerEnvFromName(DocSyncRunnerName))
assert.Equal(t, DocCleanRunnerEnv, GetRunnerEnvFromName(DocCleanRunnerName))
assert.Equal(t, DocTextRunnerEnv, GetRunnerEnvFromName(DocTextRunnerName))
assert.Equal(t, QuerySyncRunnerEnv, GetRunnerEnvFromName(QuerySyncRunnerName))
assert.Equal(t, QueryRunnerEnv, GetRunnerEnvFromName(QueryRunnerName))
assert.Equal(t, ClientSyncRunnerEnv, GetRunnerEnvFromName(ClientSyncRunnerName))
assert.Equal(t, QueryVersionSyncRunnerEnv, GetRunnerEnvFromName(QueryVersionSyncRunnerName))
assert.Equal(t, DocInitRunnerEnv, GetRunnerEnvFromName(DocInitRunnerName))
}