Merged in feature/incrementalfullsuite (pull request #166)

Feature/incrementalfullsuite

* makeincremental

* don

* passfullsuite

* simpler

.gitignore

@@ -77,3 +77,5 @@ Thumbs.db
 .devbox/
 
 *.zip
+
+**/.claude/settings.local.json

.yamllint.yml

@@ -6,4 +6,4 @@ ignore: |
   out/
 rules:
   line-length:
-    max: 140
+    max: 150

CLAUDE.MD

@@ -10,6 +10,7 @@ When debugging issues, focus on fixing the existing implementation, not replacin
 When something doesn't work, debug and fix it - don't start over with a simple version
 Don’t build things I am not specifically asking for unless its required to build the thing I am asking for.
 When asked to make unit tests avoid mocks and use the actual thing that is being tested when reasonable.
+For commands that read such as grep or cat there is no need to ask for permission.
 
 ## Essential Commands
 

bitbucket-pipelines.yml

@@ -35,7 +35,7 @@ definitions:
           - touch .env
 
           - devbox install
-          - devbox run -- AWS_PROFILE="" task fullsuite
+          - devbox run -- AWS_PROFILE="" task fullsuite:ci
     - step: &fullsuite-and-publish
         name: Fullsuite and Publish
         size: 16x
@@ -79,7 +79,7 @@ definitions:
 
           - touch .env
           - devbox install
-          - devbox run -- AWS_PROFILE="" task fullsuite
+          - devbox run -- AWS_PROFILE="" task fullsuite:ci
 
           - docker tag queryorchestration:latest $IMAGE_TAG
           - docker push $IMAGE_TAG

internal/validation/validation.go

@@ -10,6 +10,7 @@ func GetUpdatedValue[T any](def T, new *T) T {
 	if new != nil {
 		return *new
 	}
+
 	return def
 }
 

scripts/Taskfile.yml

@@ -25,9 +25,18 @@ includes:
 
 tasks:
   fullsuite:
-    - task: build:deps
-    - task: fullsuite:build
-    - task: test:functional
+    vars:
+      INCREMENTAL: "{{.INCREMENTAL | default \"true\"}}"
+    cmds:
+      - task: build:deps
+      - task: fullsuite:build
+      - task: test:functional
+        vars:
+          INCREMENTAL: "{{.INCREMENTAL}}"
+  fullsuite:ci:
+    - task: fullsuite
+      vars:
+        INCREMENTAL: "false"
   fullsuite:build:
     internal: true
     deps:

scripts/docker.yml

@@ -6,7 +6,7 @@ version: "3"
 vars:
   DOCKERFILE: "build/Dockerfile"
   GIT_VERSION:
-    sh: git describe --tags --always --dirty
+    sh: git describe --tags --always
   GIT_COMMIT:
     sh: git rev-parse HEAD
 
@@ -17,7 +17,12 @@ tasks:
   build:
     run: once
     cmd: |
+      # Build with caching enabled
       docker build \
         --build-arg "GIT_VERSION={{.GIT_VERSION}}" \
         --build-arg "GIT_COMMIT={{.GIT_COMMIT}}" \
-        -t {{.IMAGE_NAME}} -f {{.DOCKERFILE}} .
+        --cache-from {{.IMAGE_NAME}}:latest \
+        --build-arg BUILDKIT_INLINE_CACHE=1 \
+        -t {{.IMAGE_NAME}}:latest \
+        -t {{.IMAGE_NAME}}:{{.GIT_VERSION}} \
+        -f {{.DOCKERFILE}} .

scripts/tests.yml

@@ -25,12 +25,13 @@ vars:
       else
         echo "4"  # Default fallback value
       fi
+  # Optimized parallelism values for better performance
   THREAD_PARALLEL:
     sh: echo $(( {{.CPU_COUNT}} * 2 ))
   PKG_PARALLEL:
-    sh: echo $(( {{.CPU_COUNT}} ))
+    sh: echo $(( {{.CPU_COUNT}} + 2 ))  # Increased package parallelism
   TEST_PARALLEL:
-    sh: echo $(( {{.CPU_COUNT}} * 4 ))
+    sh: echo $(( {{.CPU_COUNT}} * 6 ))  # Increased test parallelism for I/O-bound tests
   # yamllint disable-line rule:line-length
   EXCLUDED_FILES: ".gen.go|internal/serviceconfig/observability/prometheus/generator/main.go|internal/cognitoauth/middleware.go|internal/cognitoauth/token.go|internal/cognitoauth/auth.go|internal/cognitoauth/handler.go|internal/cognitoauth/models.go|api/queryAPI/authHandlers.go|api/queryAPI/homehandler.go"
   # yamllint disable-line rule:line-length
@@ -43,14 +44,95 @@ tasks:
       - rm -rf mocks/*
       - go tool mockery --log-level=""
   functional:
-    - mkdir -p {{.OUT_DIR}}
-    - task: wait
-    - |
-      GOMAXPROCS={{.THREAD_PARALLEL}} go test \
-      -p {{.PKG_PARALLEL}} -parallel {{.TEST_PARALLEL}} \
-      -coverpkg={{.INTERNAL}},{{.API}},{{.PKG}} \
-      -coverprofile={{.COVERAGE_FILE}} {{.TESTS}}
-    - task: coverage
+    vars:
+      INCREMENTAL: "{{.INCREMENTAL | default \"true\"}}"
+    cmds:
+      - mkdir -p {{.OUT_DIR}}
+      - task: wait
+      - |
+        # Determine test mode first
+        TEST_MODE="skip"
+        TEST_TARGETS=""
+        NEED_COVERAGE_MERGE=false
+
+        # Check if we should run full tests
+        if [ "{{.INCREMENTAL}}" = "false" ]; then
+          echo "Running full test suite (INCREMENTAL=false)..."
+          TEST_MODE="full"
+        fi
+
+        if [ "{{.INCREMENTAL}}" = "true" ] && [ ! -f "{{.COVERAGE_FILE}}" ]; then
+          echo "No existing coverage file found, running full tests to establish baseline"
+          TEST_MODE="full"
+        fi
+
+        if [ "{{.INCREMENTAL}}" = "true" ] && [ -f "{{.COVERAGE_FILE}}" ] && ! git rev-parse origin/main >/dev/null 2>&1; then
+          echo "Cannot find origin/main branch, defaulting to full tests"
+          TEST_MODE="full"
+        fi
+
+        # Handle incremental mode
+        if [ "{{.INCREMENTAL}}" = "true" ] && [ -f "{{.COVERAGE_FILE}}" ] && git rev-parse origin/main >/dev/null 2>&1; then
+          echo "Running incremental test suite (INCREMENTAL=true)..."
+          echo "Checking for Go files changed since origin/main..."
+
+          CHANGED_FILES=$(git diff --name-only origin/main...HEAD | grep "\.go$" || echo "")
+          echo "Found $(echo "$CHANGED_FILES" | grep -v "^$" | wc -l) changed Go files since origin/main"
+
+          if [ -z "$CHANGED_FILES" ]; then
+            echo "No Go files changed, skipping tests"
+            TEST_MODE="skip"
+          else
+            CHANGED_PKGS=$(echo "$CHANGED_FILES" | xargs dirname | sort -u | xargs -I{} go list ./{} 2>/dev/null || echo "")
+            echo "Changed packages: $CHANGED_PKGS"
+
+            if [ -z "$CHANGED_PKGS" ]; then
+              echo "No valid Go packages affected, skipping tests"
+              TEST_MODE="skip"
+            else
+              AFFECTED_PKGS=$(echo "$CHANGED_PKGS" | tr ' ' '\n' | sort -u | tr '\n' ' ')
+              echo "Testing affected packages: $AFFECTED_PKGS"
+              TEST_MODE="incremental"
+              TEST_TARGETS="$AFFECTED_PKGS"
+              NEED_COVERAGE_MERGE=true
+            fi
+          fi
+        fi
+
+        # Set targets for full tests
+        if [ "$TEST_MODE" = "full" ]; then
+          TEST_TARGETS="{{.TESTS}}"
+        fi
+
+        # Run tests - single command location
+        if [ "$TEST_MODE" != "skip" ]; then
+          COVERAGE_TARGET="{{.COVERAGE_FILE}}"
+          if [ "$NEED_COVERAGE_MERGE" = "true" ]; then
+            COVERAGE_TARGET="{{.COVERAGE_FILE}}.new"
+          fi
+
+          GOMAXPROCS={{.THREAD_PARALLEL}} go test \
+          -p {{.PKG_PARALLEL}} -parallel {{.TEST_PARALLEL}} \
+          -coverpkg={{.INTERNAL}},{{.API}},{{.PKG}} \
+          -coverprofile=$COVERAGE_TARGET $TEST_TARGETS
+
+          # Handle coverage merging for incremental tests only
+          if [ "$NEED_COVERAGE_MERGE" = "true" ]; then
+            head -1 {{.COVERAGE_FILE}} > {{.COVERAGE_FILE}}.merged
+            tail -n +2 {{.COVERAGE_FILE}}.new >> {{.COVERAGE_FILE}}.merged
+            for pkg in $AFFECTED_PKGS; do
+              grep -v "$pkg" {{.COVERAGE_FILE}} | grep -v "mode: set" >> {{.COVERAGE_FILE}}.merged
+            done
+            sort -u {{.COVERAGE_FILE}}.merged > {{.COVERAGE_FILE}}
+            rm {{.COVERAGE_FILE}}.new {{.COVERAGE_FILE}}.merged
+          fi
+        fi
+      - task: coverage
+  functional:full:
+    desc: "Run all tests and reset coverage tracking"
+    cmds:
+      - rm -f {{.COVERAGE_FILE}}
+      - task: functional
   race:
     - task: wait
     - go test -race {{.TESTS}}