From e0d91d17e1300c9f9ab8c118189465a8f0f15697 Mon Sep 17 00:00:00 2001 From: Michael McGuinness Date: Wed, 11 Jun 2025 12:01:04 +0000 Subject: [PATCH] Merged in feature/incrementalfullsuite (pull request #166) Feature/incrementalfullsuite * makeincremental * don * passfullsuite * simpler --- .gitignore | 2 + .yamllint.yml | 2 +- CLAUDE.MD | 1 + bitbucket-pipelines.yml | 4 +- internal/validation/validation.go | 1 + scripts/Taskfile.yml | 15 ++++- scripts/docker.yml | 9 ++- scripts/tests.yml | 102 +++++++++++++++++++++++++++--- 8 files changed, 118 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index baccd107..7b627fdd 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,5 @@ Thumbs.db .devbox/ *.zip + +**/.claude/settings.local.json diff --git a/.yamllint.yml b/.yamllint.yml index 5ff6f7c7..fdccf109 100644 --- a/.yamllint.yml +++ b/.yamllint.yml @@ -6,4 +6,4 @@ ignore: | out/ rules: line-length: - max: 140 + max: 150 diff --git a/CLAUDE.MD b/CLAUDE.MD index 549220cd..5796eba1 100644 --- a/CLAUDE.MD +++ b/CLAUDE.MD @@ -10,6 +10,7 @@ When debugging issues, focus on fixing the existing implementation, not replacin When something doesn't work, debug and fix it - don't start over with a simple version Don’t build things I am not specifically asking for unless its required to build the thing I am asking for. When asked to make unit tests avoid mocks and use the actual thing that is being tested when reasonable. +For commands that read such as grep or cat there is no need to ask for permission. ## Essential Commands diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index f41b6736..08dc8017 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -35,7 +35,7 @@ definitions: - touch .env - devbox install - - devbox run -- AWS_PROFILE="" task fullsuite + - devbox run -- AWS_PROFILE="" task fullsuite:ci - step: &fullsuite-and-publish name: Fullsuite and Publish size: 16x @@ -79,7 +79,7 @@ definitions: - touch .env - devbox install - - devbox run -- AWS_PROFILE="" task fullsuite + - devbox run -- AWS_PROFILE="" task fullsuite:ci - docker tag queryorchestration:latest $IMAGE_TAG - docker push $IMAGE_TAG diff --git a/internal/validation/validation.go b/internal/validation/validation.go index 0a3fc132..172d4c45 100644 --- a/internal/validation/validation.go +++ b/internal/validation/validation.go @@ -10,6 +10,7 @@ func GetUpdatedValue[T any](def T, new *T) T { if new != nil { return *new } + return def } diff --git a/scripts/Taskfile.yml b/scripts/Taskfile.yml index 6ab738e1..ac260b8f 100644 --- a/scripts/Taskfile.yml +++ b/scripts/Taskfile.yml @@ -25,9 +25,18 @@ includes: tasks: fullsuite: - - task: build:deps - - task: fullsuite:build - - task: test:functional + vars: + INCREMENTAL: "{{.INCREMENTAL | default \"true\"}}" + cmds: + - task: build:deps + - task: fullsuite:build + - task: test:functional + vars: + INCREMENTAL: "{{.INCREMENTAL}}" + fullsuite:ci: + - task: fullsuite + vars: + INCREMENTAL: "false" fullsuite:build: internal: true deps: diff --git a/scripts/docker.yml b/scripts/docker.yml index 52fa1ccc..cdd75f7c 100644 --- a/scripts/docker.yml +++ b/scripts/docker.yml @@ -6,7 +6,7 @@ version: "3" vars: DOCKERFILE: "build/Dockerfile" GIT_VERSION: - sh: git describe --tags --always --dirty + sh: git describe --tags --always GIT_COMMIT: sh: git rev-parse HEAD @@ -17,7 +17,12 @@ tasks: build: run: once cmd: | + # Build with caching enabled docker build \ --build-arg "GIT_VERSION={{.GIT_VERSION}}" \ --build-arg "GIT_COMMIT={{.GIT_COMMIT}}" \ - -t {{.IMAGE_NAME}} -f {{.DOCKERFILE}} . + --cache-from {{.IMAGE_NAME}}:latest \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + -t {{.IMAGE_NAME}}:latest \ + -t {{.IMAGE_NAME}}:{{.GIT_VERSION}} \ + -f {{.DOCKERFILE}} . diff --git a/scripts/tests.yml b/scripts/tests.yml index 519e4b09..3a53fc1d 100644 --- a/scripts/tests.yml +++ b/scripts/tests.yml @@ -25,12 +25,13 @@ vars: else echo "4" # Default fallback value fi + # Optimized parallelism values for better performance THREAD_PARALLEL: sh: echo $(( {{.CPU_COUNT}} * 2 )) PKG_PARALLEL: - sh: echo $(( {{.CPU_COUNT}} )) + sh: echo $(( {{.CPU_COUNT}} + 2 )) # Increased package parallelism TEST_PARALLEL: - sh: echo $(( {{.CPU_COUNT}} * 4 )) + sh: echo $(( {{.CPU_COUNT}} * 6 )) # Increased test parallelism for I/O-bound tests # yamllint disable-line rule:line-length EXCLUDED_FILES: ".gen.go|internal/serviceconfig/observability/prometheus/generator/main.go|internal/cognitoauth/middleware.go|internal/cognitoauth/token.go|internal/cognitoauth/auth.go|internal/cognitoauth/handler.go|internal/cognitoauth/models.go|api/queryAPI/authHandlers.go|api/queryAPI/homehandler.go" # yamllint disable-line rule:line-length @@ -43,14 +44,95 @@ tasks: - rm -rf mocks/* - go tool mockery --log-level="" functional: - - mkdir -p {{.OUT_DIR}} - - task: wait - - | - GOMAXPROCS={{.THREAD_PARALLEL}} go test \ - -p {{.PKG_PARALLEL}} -parallel {{.TEST_PARALLEL}} \ - -coverpkg={{.INTERNAL}},{{.API}},{{.PKG}} \ - -coverprofile={{.COVERAGE_FILE}} {{.TESTS}} - - task: coverage + vars: + INCREMENTAL: "{{.INCREMENTAL | default \"true\"}}" + cmds: + - mkdir -p {{.OUT_DIR}} + - task: wait + - | + # Determine test mode first + TEST_MODE="skip" + TEST_TARGETS="" + NEED_COVERAGE_MERGE=false + + # Check if we should run full tests + if [ "{{.INCREMENTAL}}" = "false" ]; then + echo "Running full test suite (INCREMENTAL=false)..." + TEST_MODE="full" + fi + + if [ "{{.INCREMENTAL}}" = "true" ] && [ ! -f "{{.COVERAGE_FILE}}" ]; then + echo "No existing coverage file found, running full tests to establish baseline" + TEST_MODE="full" + fi + + if [ "{{.INCREMENTAL}}" = "true" ] && [ -f "{{.COVERAGE_FILE}}" ] && ! git rev-parse origin/main >/dev/null 2>&1; then + echo "Cannot find origin/main branch, defaulting to full tests" + TEST_MODE="full" + fi + + # Handle incremental mode + if [ "{{.INCREMENTAL}}" = "true" ] && [ -f "{{.COVERAGE_FILE}}" ] && git rev-parse origin/main >/dev/null 2>&1; then + echo "Running incremental test suite (INCREMENTAL=true)..." + echo "Checking for Go files changed since origin/main..." + + CHANGED_FILES=$(git diff --name-only origin/main...HEAD | grep "\.go$" || echo "") + echo "Found $(echo "$CHANGED_FILES" | grep -v "^$" | wc -l) changed Go files since origin/main" + + if [ -z "$CHANGED_FILES" ]; then + echo "No Go files changed, skipping tests" + TEST_MODE="skip" + else + CHANGED_PKGS=$(echo "$CHANGED_FILES" | xargs dirname | sort -u | xargs -I{} go list ./{} 2>/dev/null || echo "") + echo "Changed packages: $CHANGED_PKGS" + + if [ -z "$CHANGED_PKGS" ]; then + echo "No valid Go packages affected, skipping tests" + TEST_MODE="skip" + else + AFFECTED_PKGS=$(echo "$CHANGED_PKGS" | tr ' ' '\n' | sort -u | tr '\n' ' ') + echo "Testing affected packages: $AFFECTED_PKGS" + TEST_MODE="incremental" + TEST_TARGETS="$AFFECTED_PKGS" + NEED_COVERAGE_MERGE=true + fi + fi + fi + + # Set targets for full tests + if [ "$TEST_MODE" = "full" ]; then + TEST_TARGETS="{{.TESTS}}" + fi + + # Run tests - single command location + if [ "$TEST_MODE" != "skip" ]; then + COVERAGE_TARGET="{{.COVERAGE_FILE}}" + if [ "$NEED_COVERAGE_MERGE" = "true" ]; then + COVERAGE_TARGET="{{.COVERAGE_FILE}}.new" + fi + + GOMAXPROCS={{.THREAD_PARALLEL}} go test \ + -p {{.PKG_PARALLEL}} -parallel {{.TEST_PARALLEL}} \ + -coverpkg={{.INTERNAL}},{{.API}},{{.PKG}} \ + -coverprofile=$COVERAGE_TARGET $TEST_TARGETS + + # Handle coverage merging for incremental tests only + if [ "$NEED_COVERAGE_MERGE" = "true" ]; then + head -1 {{.COVERAGE_FILE}} > {{.COVERAGE_FILE}}.merged + tail -n +2 {{.COVERAGE_FILE}}.new >> {{.COVERAGE_FILE}}.merged + for pkg in $AFFECTED_PKGS; do + grep -v "$pkg" {{.COVERAGE_FILE}} | grep -v "mode: set" >> {{.COVERAGE_FILE}}.merged + done + sort -u {{.COVERAGE_FILE}}.merged > {{.COVERAGE_FILE}} + rm {{.COVERAGE_FILE}}.new {{.COVERAGE_FILE}}.merged + fi + fi + - task: coverage + functional:full: + desc: "Run all tests and reset coverage tracking" + cmds: + - rm -f {{.COVERAGE_FILE}} + - task: functional race: - task: wait - go test -race {{.TESTS}}