query-orchestration/scripts/tests.yml

---
# https://taskfile.dev

version: "3"

vars:
  COVERAGE_FILE: "{{.OUT_DIR}}/coverage.out"
  BENCH_FILE: "{{.OUT_DIR}}/bench.out"
  TEST_FILE: "{{.OUT_DIR}}/test.out"
  INTERNAL: "./internal/..."
  API: "./api/..."
  PKG: "./pkg/..."
  CMD: "./cmd/..."
  CPU_COUNT:
    sh: |
      # First try nproc (Linux)
      if command -v nproc >/dev/null 2>&1; then
        nproc
      # Fall back to sysctl (macOS)
      elif [ "$(uname)" = "Darwin" ]; then
        sysctl -n hw.ncpu
      # Fall back to Windows approach
      elif [ "$(uname)" = "Windows" ] || [ "$(uname)" = "MINGW"* ]; then
        echo "$NUMBER_OF_PROCESSORS"
      else
        echo "4"  # Default fallback value
      fi
  # Minimal parallelism values to avoid resource contention
  THREAD_PARALLEL:
    sh: echo "2"  # Fixed minimal value
  PKG_PARALLEL:
    sh: echo "1"  # Single package at a time
  TEST_PARALLEL:
    sh: echo "2"  # Minimal test parallelism
  # yamllint disable-line rule:line-length
  EXCLUDED_FILES: ".gen.go|.sql.go|internal/serviceconfig/observability/prometheus/generator/main.go|internal/cognitoauth/middleware.go|internal/cognitoauth/token.go|internal/cognitoauth/auth.go|internal/cognitoauth/handler.go|internal/cognitoauth/models.go|api/queryAPI/authHandlers.go|api/queryAPI/homehandler.go|api/queryAPI/controllers.go|api/queryAPI/test_helpers.go|internal/serviceconfig/common.go|internal/test/queryAPI/service.go|api/queryAPI/adminHandlers.go|internal/usermanagement/audit.go|internal/usermanagement/cognito.go|internal/usermanagement/permitio.go|internal/usermanagement/types.go"
  # yamllint disable-line rule:line-length
  TESTS: "./internal/database/repository ./test ./test/queryAPI ./test/... ./internal/serviceconfig/queue ./internal/server/... ./..."

tasks:
  mocks:generate:
    run: once
    cmds:
      - rm -rf mocks/*
      - go tool mockery --log-level=""
  functional:
    vars:
      INCREMENTAL: "{{.INCREMENTAL | default \"true\"}}"
    cmds:
      - mkdir -p {{.OUT_DIR}}
      - task: wait
      - |
        # Determine test mode first
        TEST_MODE="skip"
        TEST_TARGETS=""
        NEED_COVERAGE_MERGE=false

        # INCREMENTAL=false takes absolute precedence - force full test mode
        if [ "{{.INCREMENTAL}}" = "false" ]; then
          echo "Running full test suite (INCREMENTAL=false)..."
          TEST_MODE="full"
        else
          # Only process incremental logic if INCREMENTAL != false
          if [ ! -f "{{.COVERAGE_FILE}}" ]; then
            echo "No existing coverage file found, running full tests to establish baseline"
            TEST_MODE="full"
          elif ! git rev-parse origin/main >/dev/null 2>&1; then
            echo "Cannot find origin/main branch, defaulting to full tests"
            TEST_MODE="full"
          elif git rev-parse origin/main >/dev/null 2>&1; then
            echo "Running incremental test suite (INCREMENTAL=true)..."
            echo "Checking for Go files changed since origin/main..."

            CHANGED_FILES=$(git diff --name-only origin/main...HEAD | grep "\.go$" || echo "")
            echo "Found $(echo "$CHANGED_FILES" | grep -v "^$" | wc -l) changed Go files since origin/main"

            if [ -z "$CHANGED_FILES" ]; then
              echo "No Go files changed, skipping tests"
              TEST_MODE="skip"
            else
              CHANGED_PKGS=$(echo "$CHANGED_FILES" | xargs dirname | sort -u | xargs -I{} go list ./{} 2>/dev/null || echo "")
              echo "Changed packages: $CHANGED_PKGS"

              if [ -z "$CHANGED_PKGS" ]; then
                echo "No valid Go packages affected, skipping tests"
                TEST_MODE="skip"
              else
                AFFECTED_PKGS=$(echo "$CHANGED_PKGS" | tr ' ' '\n' | sort -u | tr '\n' ' ')
                echo "Testing affected packages: $AFFECTED_PKGS"
                TEST_MODE="incremental"
                TEST_TARGETS="$AFFECTED_PKGS"
                NEED_COVERAGE_MERGE=true
              fi
            fi
          fi
        fi

        # Set targets for full tests
        if [ "$TEST_MODE" = "full" ]; then
          TEST_TARGETS="{{.TESTS}}"
        fi

        # Run tests - single command location
        if [ "$TEST_MODE" != "skip" ]; then
          COVERAGE_TARGET="{{.COVERAGE_FILE}}"
          if [ "$NEED_COVERAGE_MERGE" = "true" ]; then
            COVERAGE_TARGET="{{.COVERAGE_FILE}}.new"
          fi

          GOMAXPROCS={{.THREAD_PARALLEL}} go test \
          -timeout 15m \
          -p {{.PKG_PARALLEL}} -parallel {{.TEST_PARALLEL}} \
          -coverpkg={{.INTERNAL}},{{.API}},{{.PKG}} \
          -coverprofile=$COVERAGE_TARGET $TEST_TARGETS

          # Handle coverage merging for incremental tests only
          if [ "$NEED_COVERAGE_MERGE" = "true" ]; then
            head -1 {{.COVERAGE_FILE}} > {{.COVERAGE_FILE}}.merged
            tail -n +2 {{.COVERAGE_FILE}}.new >> {{.COVERAGE_FILE}}.merged
            for pkg in $AFFECTED_PKGS; do
              grep -v "$pkg" {{.COVERAGE_FILE}} | grep -v "mode: set" >> {{.COVERAGE_FILE}}.merged
            done
            sort -u {{.COVERAGE_FILE}}.merged > {{.COVERAGE_FILE}}
            rm {{.COVERAGE_FILE}}.new {{.COVERAGE_FILE}}.merged
          fi
        fi
      - task: coverage
  functional:full:
    desc: "Run all tests and reset coverage tracking"
    cmds:
      - rm -f {{.COVERAGE_FILE}}
      - task: functional
  race:
    - task: wait
    - go test -race {{.TESTS}}
  graph:
    - task: wait
    - psrecord "task test:functional" --plot out/test.png --interval 0.1 --include-children
  timeline: |
    GOMAXPROCS={{.THREAD_PARALLEL}} go test \
    -p {{.PKG_PARALLEL}} -parallel {{.TEST_PARALLEL}} \
    -json -count=1 {{.TESTS}} | go run github.com/roblaszczak/vgt@latest
  wait:
    cmds:
      - docker wait $(docker ps -q --filter "label=org.testcontainers.ryuk=true") 2>/dev/null || true
      - task: prune
  prune:
    - docker container prune -f
    - docker volume prune -f
  mem:
    vars:
      NUM_RESULTS: 5
      RESULTS_FILE: "{{ .OUT_DIR }}/mem.out"
    cmds:
      - |
        YELLOW='\033[0;33m'
        BLUE='\033[0;34m'
        NC='\033[0m' # No Color

        get_memory_usage() {
            local profile_file=$1
            if [ ! -f "$profile_file" ] || [ ! -s "$profile_file" ]; then
                return
            fi

            local mem_mb=$(go tool pprof -top "{{ .RESULTS_FILE }}" 2>/dev/null | \
                          grep -E "^\s*[0-9]" | head -1 | \
                          awk '{print $1}' | \
                          sed 's/MB//g' | sed 's/KB//g' | sed 's/GB//g' | \
                          sed 's/[^0-9.]//g')

            if go tool pprof -top "{{ .RESULTS_FILE }}" 2>/dev/null | head -1 | grep -q "KB"; then
                mem_mb=$(echo "$mem_mb / 1024" | bc -l 2>/dev/null || echo "0")
            fi

            if go tool pprof -top "{{ .RESULTS_FILE }}" 2>/dev/null | head -1 | grep -q "GB"; then
                mem_mb=$(echo "$mem_mb * 1024" | bc -l 2>/dev/null || echo "0")
            fi

            echo "${mem_mb:-0}"
        }

        test_package() {
            local pkg=$1
            local pkg_short=$(basename "$pkg")
            local profile="{{ .OUT_DIR }}/mem_${pkg_short}_$(date +%s).prof"

            echo -e "${YELLOW}Testing:${NC} $pkg ... "

            # Run test with memory profiling (with timeout)
            if timeout 120 go test -memprofile="$profile" "$pkg" >/dev/null 2>&1; then
                local mem_usage=$(get_memory_usage "$profile")
                echo -e "✅ ${mem_usage} MB"
                echo "$mem_usage|$pkg" >> "{{ .RESULTS_FILE }}"
            else
                echo -e "❌ FAILED/TIMEOUT"
                echo "0|$pkg" >> "{{ .RESULTS_FILE }}"
            fi
        }

        echo -e "${BLUE}Go Test Memory Usage Analysis${NC}"
        echo -e "${BLUE}==============================${NC}"
        echo "Profile directory: out/"
        echo ""
        echo "Running tests with memory profiling..."
        echo ""

        for pkg in $(go list ./... 2>/dev/null || echo "."); do
          task test:wait
          test_package "$pkg"
        done

        if [ ! -f "{{ .RESULTS_FILE }}" ] || [ ! -s "{{ .RESULTS_FILE }}" ]; then
          exit 1
        fi

        echo ""
        echo -e "${BLUE}Memory Usage Rankings:${NC}"
        echo -e "${BLUE}=====================${NC}"

        {
            echo "MEMORY(MB)|PACKAGE"
            echo "----------|--------"
            sort -t'|' -k1 -nr "{{ .RESULTS_FILE }}" | head -{{ .NUM_RESULTS }}
        } | column -t -s'|'

        echo ""
        echo -e "${BLUE}Summary:${NC}"

        total_mem=$(awk -F'|' '{sum+=$1} END {printf "%.2f", sum}' "{{ .RESULTS_FILE }}")
        total_packages=$(wc -l < "{{ .RESULTS_FILE }}")

        echo "Total packages tested: $total_packages"
        echo "Total memory usage: ${total_mem} MB"
  perf:
    vars:
      NUM_RESULTS: 5
    cmds:
      - task: wait
      - |
        GOMAXPROCS={{.TEST_PARALLEL}} go test -parallel {{.CPU_COUNT}} -count=1 -json ./... \
        > {{.TEST_FILE}}
      - |
        echo -e "🐢 TOP {{.NUM_RESULTS}} SLOWEST INDIVIDUAL PACKAGE RUNS:"
        jq -r 'select(.Action == "pass" and .Test == null) | "\(.Elapsed)s \(.Package)"' "{{.TEST_FILE}}" | \
          sort -rn | head -n {{.NUM_RESULTS}}

        echo -e "\n📉 TOP {{.NUM_RESULTS}} SLOWEST TESTS:"
        jq -r 'select(.Action == "pass" and .Test != null) | "\(.Elapsed)s \(.Test) (\(.Package))"' "{{.TEST_FILE}}" | \
          sort -rn | head -n {{.NUM_RESULTS}}

        SLOWEST_PKG=$(jq -r 'select(.Action == "pass") | [.Package, .Elapsed] | @tsv' "{{.TEST_FILE}}" |
        awk '{pkg[$1] += $2} END {for (p in pkg) print pkg[p] "s " p}' |
        sort -rn | head -n 1 | awk '{print $2}')

        echo -e "\n🔎 TOP {{.NUM_RESULTS}} SLOWEST TESTS IN SLOWEST PACKAGE ($SLOWEST_PKG):"
        jq -r --arg pkg "$SLOWEST_PKG" \
          'select(.Action == "pass" and .Test != null and .Package == $pkg) | "\(.Elapsed)s \(.Test)"' "{{.TEST_FILE}}" |
          sort -rn | head -n {{.NUM_RESULTS}}
  bench:
    vars:
      TMP_FILE: "{{.OUT_DIR}}/coverage.tmp"
      FILTER_COVERAGE_FILE: "{{.OUT_DIR}}/filtered_coverage.out"
    cmds:
      - task: wait
      - go test -run ^Benchmark -bench=. ./... > {{.BENCH_FILE}}
      - benchstat {{.BENCH_FILE}}
  unit:short:
    deps:
      - mocks:generate
    cmds:
      - mkdir -p {{.OUT_DIR}}
      - task: wait
      - |
        go test -parallel {{.TEST_PARALLEL}} -short \
          {{.INTERNAL}} {{.API}} {{.CLI_ARGS}}
  permitio:
    desc: "Run Permit.io integration tests (requires local PDP)"
    cmds:
      - mkdir -p {{.OUT_DIR}}
      - task: wait
      - |
        echo "Running Permit.io integration tests..."
        echo "Note: Requires PERMIT_IO_API_KEY environment variable and local PDP running on port 7766"
        go test -tags=permitio -parallel {{.TEST_PARALLEL}} -count=1 \
          ./internal/cognitoauth {{.CLI_ARGS}}
  coverage:
    internal: true
    vars:
      TMP_FILE: "{{.OUT_DIR}}/coverage.tmp"
      FILTER_COVERAGE_FILE: "{{.OUT_DIR}}/filtered_coverage.out"
    cmds:
      - grep -v -E "{{.EXCLUDED_FILES}}" {{.COVERAGE_FILE}} > {{.FILTER_COVERAGE_FILE}}
      - go tool cover -func={{.FILTER_COVERAGE_FILE}} > {{.TMP_FILE}}
      - |
        COVERAGE=$(grep total: {{.TMP_FILE}} | awk '{print $NF}' | sed 's/%//')
        echo "Coverage Threshold: {{.COVERAGE_THRESHOLD}}%"
        echo "Total Coverage: $COVERAGE%"

        FAIL=false
        if (( $(echo "$COVERAGE < {{.COVERAGE_THRESHOLD}}" | bc -l) )); then
          echo "❌ Error: Total Coverage below Coverage Threshold"
          FAIL=true
        fi

        echo "Function Coverage Threshold: {{.FUNCTION_COVERAGE_THRESHOLD}}%"

        awk -v threshold="{{.FUNCTION_COVERAGE_THRESHOLD}}" '
        !/total:/ {
          line = $0;
          coverage = $NF;
          gsub(/%/, "", coverage);

          if (coverage + 0 < threshold + 0) {
            if (!printed_header) {
              print "❌ Error: The following functions have coverage below " threshold "%:";
              printed_header = 1;
            }
            print line;
            exit_code = 1;
          }
        }
        END {
          exit exit_code;
        }
        ' {{.TMP_FILE}}

        if [ $? -eq 1 ]; then
          FAIL=true
        fi

        if [ "$FAIL" = true ]; then
          exit 1
        fi

        echo "✅ All coverage checks passed!"