jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/movefromtrigger (pull request #164)

Browse Source 
Single Text Queue

* notrigger
This commit is contained in:
Michael McGuinness
2025-06-10 12:57:30 +00:00
parent 76e4f5790f
commit 693457ce2d
17 changed files with 65 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/docCleanRunner/runner_test.go
+4 -4
View File
@@ -13,7 +13,7 @@ import (
documentclean "queryorchestration/internal/document/clean"
"queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documenttexttrigger"
"queryorchestration/internal/serviceconfig/queue/documenttext"
"queryorchestration/internal/test"
objectstoremock "queryorchestration/mocks/objectstore"
@@ -26,7 +26,7 @@ import (
type DocCleanConfig struct {
runner.BaseConfig[doccleanrunner.Body]
documenttexttrigger.DocTextTriggerConfig
documenttext.DocTextConfig
objectstore.ObjectStoreConfig
}
@@ -37,7 +37,7 @@ func TestDocCleanRunner(t *testing.T) {
acfg := test.CreateAWSContainer(t, cfg)
test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint)
cfg.DocumentTextTriggerURL = test.CreateQueue(t, cfg, test.DocTextRunnerName)
cfg.DocumentTextURL = test.CreateQueue(t, cfg, test.DocTextRunnerName)
mockStore := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockStore
@@ -104,7 +104,7 @@ func TestDocCleanRunner(t *testing.T) {
}
assert.True(t, runner.Process(t.Context(), bod))
test.AssertMessageBody(t, cfg, cfg.GetDocumentTextTriggerURL(), regexp.MustCompile(fmt.Sprintf("{\"id\":\"%s\"}", docId.String())))
test.AssertMessageBody(t, cfg, cfg.GetDocumentTextURL(), regexp.MustCompile(fmt.Sprintf("{\"id\":\"%s\"}", docId.String())))
}
const pdfHelloWorld = `%PDF-1.4
build/Dockerfile
+1 -1
View File
@@ -53,5 +53,5 @@ COPY --from=build /app/bin/ .
EXPOSE 8080
HEALTHCHECK --interval=3s --timeout=3s --start-period=1s --retries=10 \
HEALTHCHECK --interval=30s --timeout=3s --start-period=1s --retries=10 \
CMD ["./healthcheck"]
cmd/docCleanRunner/main.go
+2 -2
View File
@@ -16,14 +16,14 @@ import (
documentclean "queryorchestration/internal/document/clean"
"queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documenttexttrigger"
"queryorchestration/internal/serviceconfig/queue/documenttext"
_ "github.com/lib/pq"
)
type DocCleanConfig struct {
runner.BaseConfig[doccleanrunner.Body]
documenttexttrigger.DocTextTriggerConfig
documenttext.DocTextConfig
objectstore.ObjectStoreConfig
}
cmd/docTextRunner/main.go
+2 -2
View File
@@ -22,7 +22,7 @@ import (
_ "github.com/lib/pq"
)
type DocTextTriggerConfig struct {
type DocTextConfig struct {
runner.BaseConfig[doctextrunner.Body]
textract.TextractConfig
querysync.QuerySyncConfig
@@ -32,7 +32,7 @@ type DocTextTriggerConfig struct {
func main() {
ctx := context.Background()
cfg := &DocTextTriggerConfig{}
cfg := &DocTextConfig{}
cfg.ControllerFunc = func() runner.Controller[doctextrunner.Body] {
text := documenttext.New(cfg)
deployments/compose.aws.yaml
+2 -3
View File
@@ -16,7 +16,6 @@ services:
LOG_LEVEL: DEBUG
QUEUE_URL: ${STORE_EVENT_URL}
DOCUMENT_INIT_URL: ${DOCUMENT_INIT_URL}
DOCUMENT_TEXT_PROCESS_URL: ${DOCUMENT_TEXT_PROCESS_URL}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
@@ -124,7 +123,7 @@ services:
environment:
LOG_LEVEL: DEBUG
QUEUE_URL: ${DOCUMENT_CLEAN_URL}
DOCUMENT_TEXT_TRIGGER_URL: ${DOCUMENT_TEXT_TRIGGER_URL}
DOCUMENT_TEXT_URL: ${DOCUMENT_TEXT_URL}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
@@ -160,7 +159,7 @@ services:
- 8080
environment:
LOG_LEVEL: DEBUG
QUEUE_URL: ${DOCUMENT_TEXT_TRIGGER_URL}
QUEUE_URL: ${DOCUMENT_TEXT_URL}
QUERY_SYNC_URL: ${QUERY_SYNC_URL}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
deployments/compose.local.yaml
+2 -3
View File
@@ -16,7 +16,6 @@ services:
LOG_LEVEL: DEBUG
QUEUE_URL: ${STORE_EVENT_URL}
DOCUMENT_INIT_URL: ${DOCUMENT_INIT_URL}
DOCUMENT_TEXT_PROCESS_URL: ${DOCUMENT_TEXT_PROCESS_URL}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
@@ -124,7 +123,7 @@ services:
environment:
LOG_LEVEL: DEBUG
QUEUE_URL: ${DOCUMENT_CLEAN_URL}
DOCUMENT_TEXT_TRIGGER_URL: ${DOCUMENT_TEXT_TRIGGER_URL}
DOCUMENT_TEXT_URL: ${DOCUMENT_TEXT_URL}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
@@ -159,7 +158,7 @@ services:
- 8080
environment:
LOG_LEVEL: DEBUG
QUEUE_URL: ${DOCUMENT_TEXT_TRIGGER_URL}
QUEUE_URL: ${DOCUMENT_TEXT_URL}
QUERY_SYNC_URL: ${QUERY_SYNC_URL}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
devbox.json
+2 -4
View File
@@ -20,8 +20,7 @@
"DOCUMENT_CLEAN_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_clean",
"DOCUMENT_INIT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_init",
"DOCUMENT_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_sync",
"DOCUMENT_TEXT_PROCESS_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_text_process",
"DOCUMENT_TEXT_TRIGGER_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_text_trigger",
"DOCUMENT_TEXT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_text",
"PGDATABASE": "query_orchestration",
"PGHOST": "localhost",
"PGPASSWORD": "pass",
@@ -32,8 +31,7 @@
"QNAME_DOCUMENT_CLEAN": "document_clean",
"QNAME_DOCUMENT_INIT": "document_init",
"QNAME_DOCUMENT_SYNC": "document_sync",
"QNAME_DOCUMENT_TEXT_PROCESS": "document_text_process",
"QNAME_DOCUMENT_TEXT_TRIGGER": "document_text_trigger",
"QNAME_DOCUMENT_TEXT": "document_text",
"QNAME_QUERY_RUNNER": "query_runner",
"QNAME_QUERY_SYNC": "query_sync",
"QNAME_QUERY_VERSION_SYNC": "query_version_sync",
internal/document/clean/create.go
+1 -1
View File
@@ -33,7 +33,7 @@ func (s *Service) Clean(ctx context.Context, documentId uuid.UUID) error {
func (s *Service) informClean(ctx context.Context, documentId uuid.UUID) error {
err := s.cfg.SendToQueue(ctx, &queue.SendParams{
QueueURL: s.cfg.GetDocumentTextTriggerURL(),
QueueURL: s.cfg.GetDocumentTextURL(),
Body: doctextrunner.Body{
DocumentID: documentId,
},
internal/document/clean/create_test.go
+6 -6
View File
@@ -12,7 +12,7 @@ import (
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/aws"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documenttexttrigger"
"queryorchestration/internal/serviceconfig/queue/documenttext"
objectstoremock "queryorchestration/mocks/objectstore"
queuemock "queryorchestration/mocks/queue"
@@ -27,7 +27,7 @@ import (
type DocCleanConfig struct {
aws.AWSConfig
serviceconfig.BaseConfig
documenttexttrigger.DocTextTriggerConfig
documenttext.DocTextConfig
objectstore.ObjectStoreConfig
}
@@ -40,7 +40,7 @@ func TestCreate(t *testing.T) {
cfg := &DocCleanConfig{}
cfg.QueueClient = mockSQS
cfg.DocumentTextTriggerURL = "/i/am/here"
cfg.DocumentTextURL = "/i/am/here"
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
mockS3 := objectstoremock.NewMockS3Client(t)
@@ -102,7 +102,7 @@ func TestCreate(t *testing.T) {
SendMessage(
mock.Anything,
mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
return *in.QueueUrl == cfg.DocumentTextTriggerURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", doc.ID)
return *in.QueueUrl == cfg.DocumentTextURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", doc.ID)
}),
mock.Anything,
).
@@ -143,7 +143,7 @@ func TestInformClean(t *testing.T) {
cfg := &DocCleanConfig{}
cfg.QueueClient = mockSQS
cfg.DocumentTextTriggerURL = "/i/am/here"
cfg.DocumentTextURL = "/i/am/here"
svc := Service{
cfg: cfg,
}
@@ -153,7 +153,7 @@ func TestInformClean(t *testing.T) {
SendMessage(
mock.Anything,
mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
return *in.QueueUrl == cfg.DocumentTextTriggerURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", id)
return *in.QueueUrl == cfg.DocumentTextURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", id)
}),
mock.Anything,
).
internal/document/clean/service.go
+2 -2
View File
@@ -3,12 +3,12 @@ package documentclean
import (
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documenttexttrigger"
"queryorchestration/internal/serviceconfig/queue/documenttext"
)
type ConfigProvider interface {
serviceconfig.ConfigProvider
documenttexttrigger.ConfigProvider
documenttext.ConfigProvider
objectstore.ConfigProvider
}
internal/document/clean/service_test.go
+2 -2
View File
@@ -6,14 +6,14 @@ import (
documentclean "queryorchestration/internal/document/clean"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documenttexttrigger"
"queryorchestration/internal/serviceconfig/queue/documenttext"
"github.com/stretchr/testify/assert"
)
type DocCleanConfig struct {
serviceconfig.BaseConfig
documenttexttrigger.DocTextTriggerConfig
documenttext.DocTextConfig
objectstore.ConfigProvider
}
internal/serviceconfig/queue/documenttext/config.go
+15
View File
@@ -0,0 +1,15 @@
package documenttext
const EnvName = "DOCUMENT_TEXT_URL"
type DocTextConfig struct {
DocumentTextURL string `env:"DOCUMENT_TEXT_URL,required,notEmpty"`
}
func (c *DocTextConfig) GetDocumentTextURL() string {
return c.DocumentTextURL
}
type ConfigProvider interface {
GetDocumentTextURL() string
}
internal/serviceconfig/queue/documenttext/config_test.go
+21
View File
@@ -0,0 +1,21 @@
package documenttext_test
import (
"testing"
"queryorchestration/internal/serviceconfig/queue/documenttext"
"github.com/stretchr/testify/assert"
)
func TestGetDocumentTextURL(t *testing.T) {
cfg := documenttext.DocTextConfig{}
name := cfg.GetDocumentTextURL()
assert.Equal(t, "", name)
cfg.DocumentTextURL = "name"
name = cfg.GetDocumentTextURL()
assert.Equal(t, "name", name)
assert.Equal(t, cfg.DocumentTextURL, name)
}
internal/serviceconfig/queue/documenttexttrigger/config.go
-15
View File
@@ -1,15 +0,0 @@
package documenttexttrigger
const EnvName = "DOCUMENT_TEXT_TRIGGER_URL"
type DocTextTriggerConfig struct {
DocumentTextTriggerURL string `env:"DOCUMENT_TEXT_TRIGGER_URL,required,notEmpty"`
}
func (c *DocTextTriggerConfig) GetDocumentTextTriggerURL() string {
return c.DocumentTextTriggerURL
}
type ConfigProvider interface {
GetDocumentTextTriggerURL() string
}
internal/serviceconfig/queue/documenttexttrigger/config_test.go
-21
View File
@@ -1,21 +0,0 @@
package documenttexttrigger_test
import (
"testing"
"queryorchestration/internal/serviceconfig/queue/documenttexttrigger"
"github.com/stretchr/testify/assert"
)
func TestGetDocumentTextURL(t *testing.T) {
cfg := documenttexttrigger.DocTextTriggerConfig{}
name := cfg.GetDocumentTextTriggerURL()
assert.Equal(t, "", name)
cfg.DocumentTextTriggerURL = "name"
name = cfg.GetDocumentTextTriggerURL()
assert.Equal(t, "name", name)
assert.Equal(t, cfg.DocumentTextTriggerURL, name)
}
internal/test/runner.go
+2 -2
View File
@@ -8,7 +8,7 @@ import (
"queryorchestration/internal/serviceconfig/queue/documentclean"
"queryorchestration/internal/serviceconfig/queue/documentinit"
"queryorchestration/internal/serviceconfig/queue/documentsync"
"queryorchestration/internal/serviceconfig/queue/documenttexttrigger"
"queryorchestration/internal/serviceconfig/queue/documenttext"
"queryorchestration/internal/serviceconfig/queue/query"
"queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/queue/queryversionsync"
@@ -36,7 +36,7 @@ const (
DocInitRunnerEnv RunnerEnv = documentinit.EnvName
DocSyncRunnerEnv RunnerEnv = documentsync.EnvName
DocCleanRunnerEnv RunnerEnv = documentclean.EnvName
DocTextRunnerEnv RunnerEnv = documenttexttrigger.EnvName
DocTextRunnerEnv RunnerEnv = documenttext.EnvName
QuerySyncRunnerEnv RunnerEnv = querysync.EnvName
QueryRunnerEnv RunnerEnv = query.EnvName
ClientSyncRunnerEnv RunnerEnv = clientsync.EnvName
scripts/local-deployments.yml
+1 -2
View File
@@ -73,8 +73,7 @@ tasks:
- aws sqs create-queue --queue-name $QNAME_DOCUMENT_INIT
- aws sqs create-queue --queue-name $QNAME_DOCUMENT_SYNC
- aws sqs create-queue --queue-name $QNAME_DOCUMENT_CLEAN
- aws sqs create-queue --queue-name $QNAME_DOCUMENT_TEXT_TRIGGER
- aws sqs create-queue --queue-name $QNAME_DOCUMENT_TEXT_PROCESS
- aws sqs create-queue --queue-name $QNAME_DOCUMENT_TEXT
- aws sqs create-queue --queue-name $QNAME_QUERY_SYNC
- aws sqs create-queue --queue-name $QNAME_QUERY_RUNNER
- aws sqs create-queue --queue-name $QNAME_CLIENT_SYNC