diff --git a/.dockerignore b/.dockerignore index d9bde202..4577eaf1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -20,7 +20,6 @@ **/charts **/docker-compose* **/compose.y*ml -**/Dockerfile* **/node_modules **/npm-debug.log **/obj @@ -28,12 +27,12 @@ **/values.dev.yaml LICENSE README.md +temp/ **/*_test.go cmd/*_test/ .bitbucket/ .devbox/ -build/ deployments/ mocks/ out/ diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index c6f0ca75..5daad09b 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -36,6 +36,9 @@ definitions: - devbox install - devbox run -- AWS_PROFILE="" task fullsuite + - docker save query-orchestration:latest -o dockerimage.tar + artifacts: + - dockerimage.tar pipelines: branches: diff --git a/build/Dockerfile b/build/Dockerfile index 913275d3..32a3fdd8 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -6,22 +6,24 @@ FROM golang:${GO_VERSION} AS build WORKDIR /app -RUN apt-get update && \ - apt-get install -y --no-install-recommends git=1:2.* musl-tools=1.2.* && \ - apt-get clean && \ - rm -rf /var/lib/apt-get/lists/* +RUN mkdir bin/ -COPY go.mod . -COPY go.sum . -COPY vendor/ vendor/ +RUN apt-get update && \ + apt-get install -y --no-install-recommends git=1:2.* musl-tools=1.2.* && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN go install -a -installsuffix cgo std + +COPY --link go.mod . +COPY --link go.sum . +COPY --link vendor/ vendor/ RUN go mod verify -COPY cmd/ cmd/ -COPY api/ api/ -COPY internal/ internal/ - -COPY --link .git/ .git/ +COPY --link cmd/ cmd/ +COPY --link api/ api/ +COPY --link internal/ internal/ ARG TARGETARCH @@ -29,12 +31,16 @@ ENV CGO_ENABLED=1 ENV GOARCH=${TARGETARCH} ENV CC=musl-gcc -RUN mkdir bin/ && \ - go build \ - -mod vendor \ - -tags musl \ - -ldflags "-linkmode external -extldflags -static" \ - -o bin/ ./cmd/... +ARG GIT_VERSION=dev +ARG GIT_COMMIT=unknown + +RUN go build \ + -mod vendor \ + -tags musl \ + -trimpath \ + -installsuffix cgo \ + -ldflags "-s -w -linkmode external -extldflags '-static -Wl,--as-needed -Wl,--gc-sections -Wl,-O1' -X main.version=${GIT_VERSION} -X main.gitCommit=${GIT_COMMIT}" \ + -o bin/ ./cmd/... FROM scratch AS final diff --git a/cmd/textExtractor/main.go b/cmd/textExtractor/main.go new file mode 100644 index 00000000..b3b649ca --- /dev/null +++ b/cmd/textExtractor/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "time" + + "queryorchestration/internal/serviceconfig" + "queryorchestration/internal/serviceconfig/aws" + "queryorchestration/internal/serviceconfig/objectstore" + "queryorchestration/internal/serviceconfig/queue/querysync" + "queryorchestration/internal/serviceconfig/textract" + + documenttext "queryorchestration/internal/document/text" + documenttypes "queryorchestration/internal/document/types" +) + +type Config struct { + serviceconfig.BaseConfig + textract.TextractConfig + querysync.QuerySyncConfig + objectstore.ObjectStoreConfig +} + +func main() { + ctx := context.Background() + + directory := "files/" + + cfg := &Config{} + profile := aws.Profile(os.Getenv("AWS_PROFILE")) + if profile == "" { + slog.Error("profile not available") + os.Exit(1) + } + + err := cfg.SetTextractClientWithProfile(ctx, profile) + if err != nil { + slog.Error("error creating client", "error", err) + os.Exit(1) + } + + entries, err := os.ReadDir(directory) + if err != nil { + slog.Error("error finding directory", "error", err) + os.Exit(1) + } + + for _, entry := range entries { + if entry.IsDir() { + continue + } + + extractFile(ctx, cfg, directory, entry) + } +} + +func extractFile(ctx context.Context, cfg *Config, directory string, entry os.DirEntry) { + filePath := filepath.Join(directory, entry.Name()) + pdfFile, err := os.Open(filePath) + if err != nil { + slog.Error("error opening pdf file", "error", err) + return + } + + pdf, err := documenttypes.NewPDFFromReader(pdfFile) + if err != nil { + slog.Error("error creating pdf file", "error", err) + os.Exit(1) + } + + textSvc := documenttext.New(cfg) + + startTime := time.Now() + + text, err := textSvc.GetDocumentText(ctx, pdf) + if err != nil { + slog.Error("error getting page count", "error", err) + os.Exit(1) + } + + outputFilename := fmt.Sprintf("./text/%s.txt", entry.Name()) + file, err := os.Create(outputFilename) + if err != nil { + slog.Error("error creating file", "error", err) + os.Exit(1) + } + defer file.Close() + + s, err := io.ReadAll(text) + if err != nil { + slog.Error("error reading text", "error", err) + os.Exit(1) + } + + _, err = file.Write(s) + if err != nil { + slog.Error("error writing file", "error", err) + os.Exit(1) + } + + slog.Info("extracted text", "output", outputFilename, "time", time.Since(startTime)) +} diff --git a/cmd/textractGenerator_test/textdetection.go b/cmd/textractGenerator_test/textdetection.go deleted file mode 100644 index 399c19f9..00000000 --- a/cmd/textractGenerator_test/textdetection.go +++ /dev/null @@ -1,148 +0,0 @@ -//go:build aws - -//go:generate go run -tags=aws ./textdetection.go -package main - -import ( - "context" - "encoding/json" - "fmt" - "log/slog" - "os" - documenttext "queryorchestration/internal/document/text" - documenttypes "queryorchestration/internal/document/types" - "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/serviceconfig/aws" - "queryorchestration/internal/serviceconfig/objectstore" - "queryorchestration/internal/serviceconfig/queue/querysync" - "queryorchestration/internal/serviceconfig/textract" - "sync" - "time" - - awstextract "github.com/aws/aws-sdk-go-v2/service/textract" -) - -type Config struct { - serviceconfig.BaseConfig - textract.TextractConfig - querysync.QuerySyncConfig - objectstore.ObjectStoreConfig -} - -func main() { - ctx := context.Background() - - if len(os.Args) != 2 { - fmt.Println("Error: expected one argument") - fmt.Println("Usage:", os.Args[0], "") - os.Exit(1) - } - - name := os.Args[1] - - slog.Info("arguments", "file_base", name) - - cfg := &Config{} - profile := aws.Profile(os.Getenv("AWS_PROFILE")) - if profile == "" { - slog.Error("profile not available") - os.Exit(1) - } - - err := cfg.SetTextractClientWithProfile(ctx, profile) - if err != nil { - slog.Error("error creating client", "error", err) - os.Exit(1) - } - - root, err := os.OpenRoot("assets") - if err != nil { - slog.Error("error opening root", "error", err) - os.Exit(1) - } - - pdfFile, err := root.Open(fmt.Sprintf("original/%s.pdf", name)) - if err != nil { - slog.Error("error opening pdf file", "error", err) - os.Exit(1) - } - - pdf, err := documenttypes.NewPDFFromReader(pdfFile) - if err != nil { - slog.Error("error creating pdf file", "error", err) - os.Exit(1) - } - - textSvc := documenttext.New(cfg) - - count, err := pdf.GetPageCount(ctx) - if err != nil { - slog.Error("error getting page count", "error", err) - os.Exit(1) - } - - slog.Info("detecting text", "page_count", count) - - startTime := time.Now() - allResults := []map[string]*awstextract.AnalyzeDocumentOutput{} - allResults = make([]map[string]*awstextract.AnalyzeDocumentOutput, count) - var wg sync.WaitGroup - for index := range count { - i := index - wg.Add(1) - go func() { - slog.Info("getting page", "index", i) - - baseResult, err := textSvc.GetBasePage(ctx, pdf, i) - if err != nil { - slog.Error("error detecting text", "error", err) - os.Exit(1) - } - - allResults[i] = map[string]*awstextract.AnalyzeDocumentOutput{ - "base": &baseResult, - } - - features, err := textSvc.ListRequiredFeatures(ctx, baseResult.Blocks, i) - if err != nil { - slog.Error("error listing features", "error", err) - os.Exit(1) - } - - if len(features) > 0 { - fullFeaturesResult, err := textSvc.GetPageWithFeatures(ctx, pdf, i, features) - if err != nil { - slog.Error("error getting features", "error", err) - os.Exit(1) - } - - allResults[i]["full_features"] = &fullFeaturesResult - } - - wg.Done() - }() - } - wg.Wait() - - jsonContent, err := json.MarshalIndent(allResults, "", "\t") - if err != nil { - slog.Error("error marshaling json", "error", err) - os.Exit(1) - } - - outputFilename := fmt.Sprintf("textract/%s.gen", name) - file, err := root.Create(outputFilename) - if err != nil { - slog.Error("error creating file", "error", err) - os.Exit(1) - } - defer file.Close() - - _, err = file.Write(jsonContent) - if err != nil { - slog.Error("error writing file", "error", err) - os.Exit(1) - } - - slog.Info("extracted text", "output", outputFilename, "time", time.Since(startTime)) -} diff --git a/devbox.json b/devbox.json index a76aa3f6..e47ccf4a 100644 --- a/devbox.json +++ b/devbox.json @@ -65,8 +65,7 @@ "gotestsum@latest", "goperf@latest", "graphviz@latest", - "psrecord@latest", - "linuxPackages.perf@latest" + "psrecord@latest" ], "shell": { "init_hook": [ diff --git a/devbox.lock b/devbox.lock index d26880b1..d7874f2f 100644 --- a/devbox.lock +++ b/devbox.lock @@ -645,12 +645,6 @@ } } }, - "linuxPackages.perf@latest": { - "last_modified": "2023-02-24T09:01:09Z", - "resolved": "github:NixOS/nixpkgs/7d0ed7f2e5aea07ab22ccb338d27fbe347ed2f11#linuxPackages.perf", - "source": "devbox-search", - "version": "5.15.95" - }, "nodePackages.jsonlint@1.6.3": { "last_modified": "2024-12-23T21:10:33Z", "resolved": "github:NixOS/nixpkgs/de1864217bfa9b5845f465e771e0ecb48b30e02d#nodePackages.jsonlint", diff --git a/docs/README.md b/docs/README.md index de73918c..dcdf3a2a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -136,6 +136,8 @@ It will identify the type of event and the location of said event, and forward t It supports the document initialisation and document text processing stages. Events may be pushed to the DOCINIT queue. +## textExtractor + ## cognito.auth.harness Generated by Doczy diff --git a/internal/client/update/update.go b/internal/client/update/update.go index 0033c290..d7a27e10 100644 --- a/internal/client/update/update.go +++ b/internal/client/update/update.go @@ -5,11 +5,12 @@ import ( "errors" "log/slog" - clientsyncrunner "queryorchestration/api/clientSyncRunner" "queryorchestration/internal/client" "queryorchestration/internal/database/repository" "queryorchestration/internal/serviceconfig/queue" "queryorchestration/internal/validation" + + clientsyncrunner "queryorchestration/api/clientSyncRunner" ) type Update struct { diff --git a/internal/client/update/update_test.go b/internal/client/update/update_test.go index 9a74889e..c51c1341 100644 --- a/internal/client/update/update_test.go +++ b/internal/client/update/update_test.go @@ -9,6 +9,7 @@ import ( "queryorchestration/internal/database/repository" "queryorchestration/internal/serviceconfig" "queryorchestration/internal/serviceconfig/queue/clientsync" + queuemock "queryorchestration/mocks/queue" "github.com/aws/aws-sdk-go-v2/service/sqs" diff --git a/internal/document/text/documentText.go b/internal/document/text/documentText.go index c33e3ab4..e4ee4d21 100644 --- a/internal/document/text/documentText.go +++ b/internal/document/text/documentText.go @@ -32,6 +32,8 @@ func (s *Service) GetDocumentText(ctx context.Context, doc documenttypes.File) ( textPages[index] = text + slog.Info("extracted page text", "index", index) + return nil }) } diff --git a/internal/document/upload/get_test.go b/internal/document/upload/get_test.go index 04f8927b..582ff3b3 100644 --- a/internal/document/upload/get_test.go +++ b/internal/document/upload/get_test.go @@ -2,6 +2,7 @@ package documentupload_test import ( "io" + "log" "strings" "testing" @@ -23,6 +24,7 @@ type Config struct { } func TestUpload(t *testing.T) { + t.Parallel() cfg := &Config{} test.CreateDB(t, cfg) acfg := test.CreateAWSContainer(t, cfg) @@ -42,6 +44,7 @@ func TestUpload(t *testing.T) { ClientID: "client_id", Content: strings.NewReader("abc"), } + log.Print("test") err = svc.Upload(t.Context(), file) require.NoError(t, err) diff --git a/internal/server/api/listener_test.go b/internal/server/api/listener_test.go index d635f472..26939f1a 100644 --- a/internal/server/api/listener_test.go +++ b/internal/server/api/listener_test.go @@ -7,7 +7,6 @@ import ( "net/http/httptest" "os" "strings" - "testing" "queryorchestration/internal/serviceconfig" @@ -24,6 +23,7 @@ func TestNewAPI(t *testing.T) { if testing.Short() { t.SkipNow() } + t.Parallel() ctx := context.Background() cfg := &BaseConfig{} diff --git a/internal/server/runner/listener_test.go b/internal/server/runner/listener_test.go index de1bfe0e..29e548f6 100644 --- a/internal/server/runner/listener_test.go +++ b/internal/server/runner/listener_test.go @@ -8,6 +8,7 @@ import ( "queryorchestration/internal/serviceconfig" "queryorchestration/internal/serviceconfig/objectstore" "queryorchestration/internal/test" + queuemock "queryorchestration/mocks/queue" runnermock "queryorchestration/mocks/runner" @@ -26,6 +27,7 @@ func TestNewRunner(t *testing.T) { if testing.Short() { t.SkipNow() } + t.Parallel() cfg := &TestConfig{} _ = serviceconfig.InitializeConfig(cfg) diff --git a/internal/server/runner/poll_test.go b/internal/server/runner/poll_test.go index 476e4e0e..12857fc5 100644 --- a/internal/server/runner/poll_test.go +++ b/internal/server/runner/poll_test.go @@ -8,6 +8,7 @@ import ( "queryorchestration/internal/serviceconfig/objectstore" "queryorchestration/internal/serviceconfig/queue" "queryorchestration/internal/test" + runnermock "queryorchestration/mocks/runner" "github.com/aws/aws-sdk-go-v2/service/sqs/types" @@ -20,6 +21,7 @@ import ( type MockStruct struct{} func TestPollMessages(t *testing.T) { + t.Parallel() cfg := &struct { BaseConfig[MockStruct] objectstore.ObjectStoreConfig @@ -44,6 +46,7 @@ func TestPollMessages(t *testing.T) { } func TestPollMessage(t *testing.T) { + t.Parallel() cfg := &struct { BaseConfig[MockStruct] objectstore.ObjectStoreConfig @@ -79,63 +82,97 @@ func TestPollMessage(t *testing.T) { err = ser.pollMessage(t.Context()) require.NoError(t, err) - - ares, err := cfg.ReceiveFromQueue(t.Context(), &queue.ReceiveParams{ - QueueURL: cfg.GetQueueURL(), - }) - require.NoError(t, err) - require.Len(t, ares.Messages, 0) } func TestProcessMessage(t *testing.T) { - cfg := &struct { - BaseConfig[MockStruct] - objectstore.ObjectStoreConfig - }{} - acfg := test.CreateAWSContainer(t, cfg) + t.Run("works", func(t *testing.T) { + t.Parallel() + cfg := &struct { + BaseConfig[MockStruct] + objectstore.ObjectStoreConfig + }{} + acfg := test.CreateAWSContainer(t, cfg) - test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint) - cfg.QueueURL = test.CreateQueue(t, cfg, test.QueryRunnerName) + test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint) + cfg.QueueURL = test.CreateQueue(t, cfg, test.QueryRunnerName) - mockController := runnermock.NewMockController[MockStruct](t) - cfg.Controller = mockController - cfg.Validator = validator.New() + mockController := runnermock.NewMockController[MockStruct](t) + cfg.Controller = mockController + cfg.Validator = validator.New() - ser := &Server[MockStruct]{ - cfg: cfg, - cleanup: func() error { return nil }, - } + ser := &Server[MockStruct]{ + cfg: cfg, + cleanup: func() error { return nil }, + } - mockController.EXPECT(). - Process( - mock.Anything, - mock.MatchedBy(func(in MockStruct) bool { - return true - }), - ). - Return(true) + err := cfg.SendToQueue(t.Context(), &queue.SendParams{ + QueueURL: cfg.GetQueueURL(), + Body: struct{}{}, + }) + require.NoError(t, err) - err := cfg.SendToQueue(t.Context(), &queue.SendParams{ - QueueURL: cfg.GetQueueURL(), - Body: struct{}{}, + msg := test.AssertMessage(t, cfg, &queue.ReceiveParams{ + QueueURL: cfg.GetQueueURL(), + }) + + mockController.EXPECT(). + Process( + mock.Anything, + mock.MatchedBy(func(in MockStruct) bool { + return true + }), + ). + Return(true) + + err = ser.processMessage(t.Context(), &msg) + require.NoError(t, err) }) - require.NoError(t, err) + t.Run("fail Controller", func(t *testing.T) { + t.Parallel() + cfg := &struct { + BaseConfig[MockStruct] + objectstore.ObjectStoreConfig + }{} + acfg := test.CreateAWSContainer(t, cfg) - msg := test.AssertMessage(t, cfg, &queue.ReceiveParams{ - QueueURL: cfg.GetQueueURL(), + test.SetQueueClient(t, t.Context(), cfg, acfg.ExternalEndpoint) + cfg.QueueURL = test.CreateQueue(t, cfg, test.QueryRunnerName) + + mockController := runnermock.NewMockController[MockStruct](t) + cfg.Controller = mockController + cfg.Validator = validator.New() + + ser := &Server[MockStruct]{ + cfg: cfg, + cleanup: func() error { return nil }, + } + + err := cfg.SendToQueue(t.Context(), &queue.SendParams{ + QueueURL: cfg.GetQueueURL(), + Body: struct{}{}, + }) + require.NoError(t, err) + + msg := test.AssertMessage(t, cfg, &queue.ReceiveParams{ + QueueURL: cfg.GetQueueURL(), + }) + + mockController.EXPECT(). + Process( + mock.Anything, + mock.MatchedBy(func(in MockStruct) bool { + return true + }), + ). + Return(false) + + err = ser.processMessage(t.Context(), &msg) + require.Error(t, err) }) - - err = ser.processMessage(t.Context(), &msg) - require.NoError(t, err) - - ares, err := cfg.ReceiveFromQueue(t.Context(), &queue.ReceiveParams{ - QueueURL: cfg.GetQueueURL(), - }) - require.NoError(t, err) - require.Len(t, ares.Messages, 0) } func TestProcessBody(t *testing.T) { + t.Parallel() type MockStruct struct { Example string `json:"example" validate:"required"` } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index c4c526ee..224e8cc9 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -15,6 +15,7 @@ import ( ) func TestNewServer(t *testing.T) { + t.Parallel() if testing.Short() { t.SkipNow() } diff --git a/internal/serviceconfig/queue/config_test.go b/internal/serviceconfig/queue/config_test.go index 93dfe995..1fa9dcbe 100644 --- a/internal/serviceconfig/queue/config_test.go +++ b/internal/serviceconfig/queue/config_test.go @@ -30,6 +30,7 @@ func TestSetQueueClient(t *testing.T) { } func TestPingQueueByURL(t *testing.T) { + t.Parallel() cfg := &AWSConfig{} acfg := test.CreateAWSContainer(t, cfg) @@ -39,6 +40,7 @@ func TestPingQueueByURL(t *testing.T) { err := cfg.PingQueueByURL(t.Context(), cfg.QueryURL) require.NoError(t, err) } + func TestGetSQSEndpoint(t *testing.T) { cfg := queue.QueueConfig{} diff --git a/internal/serviceconfig/queue/delete_test.go b/internal/serviceconfig/queue/delete_test.go index 8772ad35..ad917c08 100644 --- a/internal/serviceconfig/queue/delete_test.go +++ b/internal/serviceconfig/queue/delete_test.go @@ -10,6 +10,7 @@ import ( ) func TestDelete(t *testing.T) { + t.Parallel() cfg := &AWSConfig{} acfg := test.CreateAWSContainer(t, cfg) @@ -18,8 +19,7 @@ func TestDelete(t *testing.T) { err := cfg.SendToQueue(t.Context(), &queue.SendParams{ QueueURL: cfg.GetQueryURL(), - Body: struct { - }{}, + Body: struct{}{}, }) require.NoError(t, err) diff --git a/internal/serviceconfig/queue/receive_test.go b/internal/serviceconfig/queue/receive_test.go index d764c7bb..b0adc98b 100644 --- a/internal/serviceconfig/queue/receive_test.go +++ b/internal/serviceconfig/queue/receive_test.go @@ -11,6 +11,7 @@ import ( ) func TestReceive(t *testing.T) { + t.Parallel() cfg := &AWSConfig{} acfg := test.CreateAWSContainer(t, cfg) diff --git a/internal/test/network.go b/internal/test/network.go index 3a0e030b..e0a1c185 100644 --- a/internal/test/network.go +++ b/internal/test/network.go @@ -15,16 +15,14 @@ const ( networkName = "queryorchestration_test" ) -var ( - networkOnce sync.Once -) +var networkOnce sync.Once func GetNetwork(t testing.TB) string { networkOnce.Do(func() { - cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) - require.NoError(t, err) + cli := getDockerClient(t) + defer cli.Close() - _, err = cli.NetworkCreate(t.Context(), networkName, network.CreateOptions{ + _, err := cli.NetworkCreate(t.Context(), networkName, network.CreateOptions{ Driver: network.NetworkBridge, }) conflictErrorMsg := fmt.Sprintf("Error response from daemon: network with name %s already exists", networkName) @@ -37,3 +35,9 @@ func GetNetwork(t testing.TB) string { return networkName } + +func getDockerClient(t testing.TB) *client.Client { + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + require.NoError(t, err) + return cli +} diff --git a/scripts/Taskfile.yml b/scripts/Taskfile.yml index 10345246..6ab738e1 100644 --- a/scripts/Taskfile.yml +++ b/scripts/Taskfile.yml @@ -26,14 +26,14 @@ includes: tasks: fullsuite: - task: build:deps - - task: build - - task: fullsuite:test - fullsuite:test: + - task: fullsuite:build + - task: test:functional + fullsuite:build: internal: true deps: - - task: test:functional + - task: build - task: lint - graph: psrecord "task fullsuite" --plot out/fullsuite.png --interval 0.1 --include-children + fullsuite:graph: psrecord "task fullsuite" --plot out/fullsuite.png --interval 0.1 --include-children precommit: - task: lint deps:tidy: @@ -46,6 +46,7 @@ tasks: task: docker:build build:deps: internal: true + run: once cmds: - task: deps:tidy - task: generate diff --git a/scripts/docker.yml b/scripts/docker.yml index 53f3093e..52fa1ccc 100644 --- a/scripts/docker.yml +++ b/scripts/docker.yml @@ -5,6 +5,10 @@ version: "3" vars: DOCKERFILE: "build/Dockerfile" + GIT_VERSION: + sh: git describe --tags --always --dirty + GIT_COMMIT: + sh: git rev-parse HEAD tasks: lint: @@ -12,5 +16,8 @@ tasks: - hadolint {{.DOCKERFILE}} -t none build: run: once - cmds: - - docker build -t {{.IMAGE_NAME}} -f {{.DOCKERFILE}} . + cmd: | + docker build \ + --build-arg "GIT_VERSION={{.GIT_VERSION}}" \ + --build-arg "GIT_COMMIT={{.GIT_COMMIT}}" \ + -t {{.IMAGE_NAME}} -f {{.DOCKERFILE}} . diff --git a/scripts/tests.yml b/scripts/tests.yml index 2e9c6f11..519e4b09 100644 --- a/scripts/tests.yml +++ b/scripts/tests.yml @@ -25,10 +25,16 @@ vars: else echo "4" # Default fallback value fi - TEST_PARALLEL: + THREAD_PARALLEL: + sh: echo $(( {{.CPU_COUNT}} * 2 )) + PKG_PARALLEL: sh: echo $(( {{.CPU_COUNT}} )) + TEST_PARALLEL: + sh: echo $(( {{.CPU_COUNT}} * 4 )) # yamllint disable-line rule:line-length EXCLUDED_FILES: ".gen.go|internal/serviceconfig/observability/prometheus/generator/main.go|internal/cognitoauth/middleware.go|internal/cognitoauth/token.go|internal/cognitoauth/auth.go|internal/cognitoauth/handler.go|internal/cognitoauth/models.go|api/queryAPI/authHandlers.go|api/queryAPI/homehandler.go" + # yamllint disable-line rule:line-length + TESTS: "./internal/database/repository ./test ./test/queryAPI ./test/... ./internal/serviceconfig/queue ./internal/server/... ./..." tasks: mocks:generate: @@ -40,14 +46,28 @@ tasks: - mkdir -p {{.OUT_DIR}} - task: wait - | - GOMAXPROCS={{.TEST_PARALLEL}} go test -parallel {{.CPU_COUNT}} \ + GOMAXPROCS={{.THREAD_PARALLEL}} go test \ + -p {{.PKG_PARALLEL}} -parallel {{.TEST_PARALLEL}} \ -coverpkg={{.INTERNAL}},{{.API}},{{.PKG}} \ - -coverprofile={{.COVERAGE_FILE}} \ - ./... + -coverprofile={{.COVERAGE_FILE}} {{.TESTS}} - task: coverage - race: GOMAXPROCS={{.TEST_PARALLEL}} go test -parallel {{.CPU_COUNT}} -race ./... - graph: psrecord "task test:functional" --plot out/test.png --interval 0.1 --include-children - wait: docker wait $(docker ps -q --filter "label=org.testcontainers.ryuk=true") 2>/dev/null || true + race: + - task: wait + - go test -race {{.TESTS}} + graph: + - task: wait + - psrecord "task test:functional" --plot out/test.png --interval 0.1 --include-children + timeline: | + GOMAXPROCS={{.THREAD_PARALLEL}} go test \ + -p {{.PKG_PARALLEL}} -parallel {{.TEST_PARALLEL}} \ + -json -count=1 {{.TESTS}} | go run github.com/roblaszczak/vgt@latest + wait: + cmds: + - docker wait $(docker ps -q --filter "label=org.testcontainers.ryuk=true") 2>/dev/null || true + - task: prune + prune: + - docker container prune -f + - docker volume prune -f mem: vars: NUM_RESULTS: 5 @@ -137,6 +157,7 @@ tasks: vars: NUM_RESULTS: 5 cmds: + - task: wait - | GOMAXPROCS={{.TEST_PARALLEL}} go test -parallel {{.CPU_COUNT}} -count=1 -json ./... \ > {{.TEST_FILE}} @@ -162,6 +183,7 @@ tasks: TMP_FILE: "{{.OUT_DIR}}/coverage.tmp" FILTER_COVERAGE_FILE: "{{.OUT_DIR}}/filtered_coverage.out" cmds: + - task: wait - go test -run ^Benchmark -bench=. ./... > {{.BENCH_FILE}} - benchstat {{.BENCH_FILE}} unit:short: @@ -169,6 +191,7 @@ tasks: - mocks:generate cmds: - mkdir -p {{.OUT_DIR}} + - task: wait - | go test -parallel {{.TEST_PARALLEL}} -short \ {{.INTERNAL}} {{.API}} {{.CLI_ARGS}} diff --git a/test/process_test.go b/test/process_test.go index 09003c26..4654edc2 100644 --- a/test/process_test.go +++ b/test/process_test.go @@ -38,7 +38,7 @@ func TestProcess(t *testing.T) { var jsonId queryapi.QueryID var textractExpectation test.MockExpectation textractBody := "Hello World" - wg.Add(3) + wg.Add(2) go func(t testing.TB) { _, jsonId = queryapitest.CreateDependentQueries(t, net.Client) wg.Done() @@ -47,6 +47,10 @@ func TestProcess(t *testing.T) { clientId = queryapitest.CreateClientWithSync(t, net.Client) wg.Done() }(t) + + wg.Wait() + + wg.Add(3) go func(t testing.TB) { textractExpectation = test.CreateDetectDocumentTextExpectation( t, @@ -55,17 +59,21 @@ func TestProcess(t *testing.T) { ) wg.Done() }(t) + go func(t testing.TB) { + queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId) + wg.Done() + }(t) + go func(t testing.TB) { + queryapitest.CreateFile(t, net.Client, queryapitest.File{ + Filename: "helloworld.pdf", + Content: []byte(pdfHelloWorld), + ClientID: clientId, + }) + wg.Done() + }(t) wg.Wait() - queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId) - - queryapitest.CreateFile(t, net.Client, queryapitest.File{ - Filename: "helloworld.pdf", - Content: []byte(pdfHelloWorld), - ClientID: clientId, - }) - test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request) part := uint16(0) @@ -117,7 +125,7 @@ func TestProcess(t *testing.T) { var jsonId queryapi.QueryID var textractExpectation test.MockExpectation textractBody := "Hello World" - wg.Add(3) + wg.Add(2) go func(t testing.TB) { _, jsonId = queryapitest.CreateDependentQueries(t, net.Client) wg.Done() @@ -126,6 +134,22 @@ func TestProcess(t *testing.T) { clientId = queryapitest.CreateClientWithSync(t, net.Client) wg.Done() }(t) + + wg.Wait() + + wg.Add(3) + go func(t testing.TB) { + queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId) + wg.Done() + }(t) + go func(t testing.TB) { + queryapitest.CreateFile(t, net.Client, queryapitest.File{ + Filename: "helloworld.pdf", + Content: []byte(pdfHelloWorld), + ClientID: clientId, + }) + wg.Done() + }(t) go func(t testing.TB) { textractExpectation = test.CreateDetectDocumentTextExpectation( t, @@ -137,14 +161,6 @@ func TestProcess(t *testing.T) { wg.Wait() - queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId) - - queryapitest.CreateFile(t, net.Client, queryapitest.File{ - Filename: "helloworld.pdf", - Content: []byte(pdfHelloWorld), - ClientID: clientId, - }) - test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request) queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC) @@ -191,7 +207,7 @@ func TestProcess(t *testing.T) { var jsonId queryapi.QueryID var textractExpectation test.MockExpectation textractBody := "Hello World" - wg.Add(3) + wg.Add(2) go func(t testing.TB) { _, jsonId = queryapitest.CreateDependentQueries(t, net.Client) @@ -207,6 +223,10 @@ func TestProcess(t *testing.T) { clientId = queryapitest.CreateClientWithSync(t, net.Client) wg.Done() }(t) + + wg.Wait() + + wg.Add(3) go func(t testing.TB) { textractExpectation = test.CreateDetectDocumentTextExpectation( t, @@ -215,17 +235,21 @@ func TestProcess(t *testing.T) { ) wg.Done() }(t) + go func(t testing.TB) { + queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId) + wg.Done() + }(t) + go func(t testing.TB) { + queryapitest.CreateFile(t, net.Client, queryapitest.File{ + Filename: "helloworld.pdf", + Content: []byte(pdfHelloWorld), + ClientID: clientId, + }) + wg.Done() + }(t) wg.Wait() - queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId) - - queryapitest.CreateFile(t, net.Client, queryapitest.File{ - Filename: "helloworld.pdf", - Content: []byte(pdfHelloWorld), - ClientID: clientId, - }) - test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request) queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)