Merged in feature/track-filesize (pull request #200)

track file sizes for all documents in system * feature complete needs dev testing
2026-01-12 17:46:07 +00:00
parent 4ad8168f35
commit ebf47c6013
37 changed files with 1514 additions and 374 deletions
@@ -0,0 +1,152 @@
+// Package test provides configuration utilities for test infrastructure.
+// This file centralizes timeout configuration for CI/CD stability.
+package test
+
+import (
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	"time"
+)
+
+// GetHTTPTimeout returns the HTTP client timeout for API calls.
+// Default is 30 seconds, configurable via E2E_HTTP_TIMEOUT environment variable.
+// In CI environments, this timeout ensures that stalled HTTP calls don't block
+// indefinitely, which was a root cause of test hangs.
+func GetHTTPTimeout() time.Duration {
+	if t := os.Getenv("E2E_HTTP_TIMEOUT"); t != "" {
+		if d, err := time.ParseDuration(t); err == nil {
+			return d
+		}
+	}
+	return 30 * time.Second
+}
+
+// GetPollingTimeout returns the timeout for polling operations like waiting
+// for client status or mock endpoint verification.
+// Default is 60 seconds locally, 180 seconds in CI (detected via CI or
+// BITBUCKET_PIPELINE_UUID environment variables).
+// Configurable via E2E_WAIT_TIMEOUT environment variable.
+func GetPollingTimeout() time.Duration {
+	if t := os.Getenv("E2E_WAIT_TIMEOUT"); t != "" {
+		if d, err := time.ParseDuration(t); err == nil {
+			return d
+		}
+	}
+	if isCI() {
+		return 180 * time.Second // 3 minutes in CI
+	}
+	return 60 * time.Second // 1 minute locally
+}
+
+// GetPollInterval returns the interval between polling attempts.
+// Default is 500ms, configurable via E2E_POLL_INTERVAL environment variable.
+func GetPollInterval() time.Duration {
+	if t := os.Getenv("E2E_POLL_INTERVAL"); t != "" {
+		if d, err := time.ParseDuration(t); err == nil {
+			return d
+		}
+	}
+	return 500 * time.Millisecond
+}
+
+// GetHTTPClient returns an HTTP client with the configured timeout.
+// This should be used instead of &http.Client{} to ensure all HTTP operations
+// have bounded timeouts and won't hang indefinitely.
+func GetHTTPClient() *http.Client {
+	return &http.Client{
+		Timeout: GetHTTPTimeout(),
+	}
+}
+
+// isCI returns true if running in a CI environment.
+// Detects CI via the CI or BITBUCKET_PIPELINE_UUID environment variables.
+func isCI() bool {
+	return os.Getenv("CI") != "" || os.Getenv("BITBUCKET_PIPELINE_UUID") != ""
+}
+
+// WaitForExternalReadiness waits for a service to be reachable from outside Docker.
+// This addresses the race condition where a container's internal health check passes
+// but the external port mapping isn't yet ready. This is particularly important in
+// Docker-in-Docker (DinD) environments like Bitbucket Pipelines.
+//
+// Parameters:
+//   - uri: The base URI of the service (e.g., "http://localhost:32768")
+//   - healthPath: The health check endpoint path (e.g., "/health" or "/liveness/probe")
+//   - timeout: Maximum time to wait for the service to become reachable
+//
+// Returns an error if the service is not reachable within the timeout period.
+func WaitForExternalReadiness(uri string, healthPath string, timeout time.Duration) error {
+	client := &http.Client{Timeout: 2 * time.Second}
+	deadline := time.Now().Add(timeout)
+
+	fullURL := fmt.Sprintf("%s%s", uri, healthPath)
+	slog.Info("Waiting for external readiness", "url", fullURL, "timeout", timeout)
+
+	var lastErr error
+	attempts := 0
+	for time.Now().Before(deadline) {
+		attempts++
+		resp, err := client.Get(fullURL)
+		if err == nil && resp.StatusCode == http.StatusOK {
+			resp.Body.Close()
+			slog.Info("Service is externally reachable", "url", fullURL, "attempts", attempts)
+			return nil
+		}
+		if err != nil {
+			lastErr = err
+		} else {
+			lastErr = fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+			resp.Body.Close()
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+
+	return fmt.Errorf("service not reachable from test client after %v (%d attempts): %w", timeout, attempts, lastErr)
+}
+
+// WaitForAPIRoutes waits for API routes to be fully registered.
+// This addresses a race condition where the health endpoint returns 200 but other
+// API routes haven't been registered yet, causing "404 page not found" errors.
+// This is particularly important in slow CI environments where route registration
+// may take longer after the health endpoint becomes available.
+//
+// Parameters:
+//   - uri: The base URI of the service (e.g., "http://localhost:32768")
+//   - timeout: Maximum time to wait for routes to be registered
+//
+// Returns an error if routes are not available within the timeout period.
+func WaitForAPIRoutes(uri string, timeout time.Duration) error {
+	client := &http.Client{Timeout: 2 * time.Second}
+	deadline := time.Now().Add(timeout)
+
+	// Check /query endpoint - should return 200 with empty array when routes are ready
+	// A "404 page not found" means routes aren't registered yet
+	routeURL := fmt.Sprintf("%s/query", uri)
+	slog.Info("Waiting for API routes to be registered", "url", routeURL, "timeout", timeout)
+
+	var lastErr error
+	attempts := 0
+	for time.Now().Before(deadline) {
+		attempts++
+		resp, err := client.Get(routeURL)
+		if err == nil {
+			// Any response other than connection error means routes are registered
+			// 200 = success, 401 = auth required (but route exists), etc.
+			// Only "404 page not found" indicates routes not yet registered
+			if resp.StatusCode != http.StatusNotFound {
+				resp.Body.Close()
+				slog.Info("API routes are registered", "url", routeURL, "attempts", attempts, "status", resp.StatusCode)
+				return nil
+			}
+			lastErr = fmt.Errorf("routes not registered yet: %d", resp.StatusCode)
+			resp.Body.Close()
+		} else {
+			lastErr = err
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+
+	return fmt.Errorf("API routes not registered after %v (%d attempts): %w", timeout, attempts, lastErr)
+}
@@ -0,0 +1,219 @@
+package test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetHTTPTimeout(t *testing.T) {
+	// Test default value
+	timeout := GetHTTPTimeout()
+	assert.Equal(t, 30*time.Second, timeout, "default HTTP timeout should be 30 seconds")
+
+	// Test with environment variable
+	t.Setenv("E2E_HTTP_TIMEOUT", "45s")
+	timeout = GetHTTPTimeout()
+	assert.Equal(t, 45*time.Second, timeout, "HTTP timeout should respect E2E_HTTP_TIMEOUT env var")
+
+	// Test with invalid environment variable (should fall back to default)
+	t.Setenv("E2E_HTTP_TIMEOUT", "invalid")
+	timeout = GetHTTPTimeout()
+	assert.Equal(t, 30*time.Second, timeout, "invalid E2E_HTTP_TIMEOUT should fall back to default")
+}
+
+func TestGetPollingTimeout(t *testing.T) {
+	// Clear CI env vars to ensure we test non-CI default first
+	t.Setenv("CI", "")
+	t.Setenv("BITBUCKET_PIPELINE_UUID", "")
+
+	// Test default value (non-CI)
+	timeout := GetPollingTimeout()
+	assert.Equal(t, 60*time.Second, timeout, "default polling timeout should be 60 seconds")
+
+	// Test with environment variable override
+	t.Setenv("E2E_WAIT_TIMEOUT", "120s")
+	timeout = GetPollingTimeout()
+	assert.Equal(t, 120*time.Second, timeout, "polling timeout should respect E2E_WAIT_TIMEOUT env var")
+
+	// Test CI detection
+	t.Setenv("E2E_WAIT_TIMEOUT", "")
+	t.Setenv("CI", "true")
+	timeout = GetPollingTimeout()
+	assert.Equal(t, 180*time.Second, timeout, "polling timeout in CI should be 180 seconds")
+
+	// Test BITBUCKET_PIPELINE_UUID detection
+	t.Setenv("CI", "")
+	t.Setenv("BITBUCKET_PIPELINE_UUID", "some-uuid")
+	timeout = GetPollingTimeout()
+	assert.Equal(t, 180*time.Second, timeout, "polling timeout with BITBUCKET_PIPELINE_UUID should be 180 seconds")
+}
+
+func TestGetPollInterval(t *testing.T) {
+	// Test default value
+	interval := GetPollInterval()
+	assert.Equal(t, 500*time.Millisecond, interval, "default poll interval should be 500ms")
+
+	// Test with environment variable
+	t.Setenv("E2E_POLL_INTERVAL", "1s")
+	interval = GetPollInterval()
+	assert.Equal(t, 1*time.Second, interval, "poll interval should respect E2E_POLL_INTERVAL env var")
+
+	// Test with invalid environment variable (should fall back to default)
+	t.Setenv("E2E_POLL_INTERVAL", "invalid")
+	interval = GetPollInterval()
+	assert.Equal(t, 500*time.Millisecond, interval, "invalid E2E_POLL_INTERVAL should fall back to default")
+}
+
+func TestGetHTTPClient(t *testing.T) {
+	client := GetHTTPClient()
+	assert.NotNil(t, client, "GetHTTPClient should return a non-nil client")
+	assert.Equal(t, GetHTTPTimeout(), client.Timeout, "HTTP client timeout should match GetHTTPTimeout()")
+}
+
+func TestIsCI(t *testing.T) {
+	// Clear CI env vars to ensure consistent test behavior
+	t.Setenv("CI", "")
+	t.Setenv("BITBUCKET_PIPELINE_UUID", "")
+
+	// Test non-CI environment
+	assert.False(t, isCI(), "isCI should return false when CI env vars are not set")
+
+	// Test with CI env var
+	t.Setenv("CI", "true")
+	assert.True(t, isCI(), "isCI should return true when CI is set")
+
+	// Test with BITBUCKET_PIPELINE_UUID
+	t.Setenv("CI", "")
+	t.Setenv("BITBUCKET_PIPELINE_UUID", "some-uuid")
+	assert.True(t, isCI(), "isCI should return true when BITBUCKET_PIPELINE_UUID is set")
+}
+
+func TestWaitForExternalReadiness(t *testing.T) {
+	t.Run("success on first attempt", func(t *testing.T) {
+		// Create a test server that returns 200 OK
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.URL.Path == "/health" {
+				w.WriteHeader(http.StatusOK)
+				return
+			}
+			w.WriteHeader(http.StatusNotFound)
+		}))
+		defer server.Close()
+
+		err := WaitForExternalReadiness(server.URL, "/health", 5*time.Second)
+		assert.NoError(t, err, "should succeed when server is immediately ready")
+	})
+
+	t.Run("success after delay", func(t *testing.T) {
+		attempts := 0
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.URL.Path == "/health" {
+				attempts++
+				if attempts < 3 {
+					w.WriteHeader(http.StatusServiceUnavailable)
+					return
+				}
+				w.WriteHeader(http.StatusOK)
+				return
+			}
+			w.WriteHeader(http.StatusNotFound)
+		}))
+		defer server.Close()
+
+		err := WaitForExternalReadiness(server.URL, "/health", 5*time.Second)
+		assert.NoError(t, err, "should succeed after retries")
+		assert.GreaterOrEqual(t, attempts, 3, "should have made multiple attempts")
+	})
+
+	t.Run("timeout on non-responsive server", func(t *testing.T) {
+		// Create a server that always returns 503
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusServiceUnavailable)
+		}))
+		defer server.Close()
+
+		err := WaitForExternalReadiness(server.URL, "/health", 2*time.Second)
+		assert.Error(t, err, "should fail when server never becomes ready")
+		assert.Contains(t, err.Error(), "not reachable", "error should indicate service not reachable")
+	})
+
+	t.Run("timeout on unreachable server", func(t *testing.T) {
+		// Use an invalid URL that will fail to connect
+		err := WaitForExternalReadiness("http://127.0.0.1:1", "/health", 2*time.Second)
+		assert.Error(t, err, "should fail when server is unreachable")
+		assert.Contains(t, err.Error(), "not reachable", "error should indicate service not reachable")
+	})
+}
+
+func TestWaitForAPIRoutes(t *testing.T) {
+	t.Run("success when routes return 200", func(t *testing.T) {
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.URL.Path == "/query" {
+				w.WriteHeader(http.StatusOK)
+				_, _ = w.Write([]byte("[]"))
+				return
+			}
+			w.WriteHeader(http.StatusNotFound)
+		}))
+		defer server.Close()
+
+		err := WaitForAPIRoutes(server.URL, 5*time.Second)
+		assert.NoError(t, err, "should succeed when routes return 200")
+	})
+
+	t.Run("success after routes become available", func(t *testing.T) {
+		attempts := 0
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.URL.Path == "/query" {
+				attempts++
+				if attempts < 3 {
+					// Simulate "404 page not found" before routes are registered
+					w.WriteHeader(http.StatusNotFound)
+					_, _ = w.Write([]byte("404 page not found"))
+					return
+				}
+				w.WriteHeader(http.StatusOK)
+				_, _ = w.Write([]byte("[]"))
+				return
+			}
+			w.WriteHeader(http.StatusNotFound)
+		}))
+		defer server.Close()
+
+		err := WaitForAPIRoutes(server.URL, 5*time.Second)
+		assert.NoError(t, err, "should succeed after routes become available")
+		assert.GreaterOrEqual(t, attempts, 3, "should have made multiple attempts")
+	})
+
+	t.Run("success when routes return 401", func(t *testing.T) {
+		// 401 means the route exists but requires auth - route is registered
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.URL.Path == "/query" {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+			w.WriteHeader(http.StatusNotFound)
+		}))
+		defer server.Close()
+
+		err := WaitForAPIRoutes(server.URL, 5*time.Second)
+		assert.NoError(t, err, "should succeed when routes return 401 (route exists)")
+	})
+
+	t.Run("timeout when routes never register", func(t *testing.T) {
+		// Server always returns 404 - routes never registered
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusNotFound)
+			_, _ = w.Write([]byte("404 page not found"))
+		}))
+		defer server.Close()
+
+		err := WaitForAPIRoutes(server.URL, 2*time.Second)
+		assert.Error(t, err, "should fail when routes never register")
+		assert.Contains(t, err.Error(), "not registered", "error should indicate routes not registered")
+	})
+}
@@ -70,7 +70,24 @@ func CreateFullNetwork(t testing.TB, ctx context.Context, cfg FullDependenciesCo

 	wg.Wait()

-	qService, err := queryapi.NewClientWithResponses(apiContainers[QueryAPIName].URI)
+	// Wait for external readiness before creating the client.
+	// This addresses the race condition where containers report healthy internally
+	// but the external port mapping isn't fully ready, causing 404 errors in CI.
+	err := WaitForExternalReadiness(apiContainers[QueryAPIName].URI, "/health", GetPollingTimeout())
+	require.NoError(t, err, "QueryAPI not externally reachable")
+
+	// Wait for API routes to be registered.
+	// The health endpoint may return 200 before all routes are registered,
+	// causing "404 page not found" errors on actual API calls.
+	err = WaitForAPIRoutes(apiContainers[QueryAPIName].URI, GetPollingTimeout())
+	require.NoError(t, err, "QueryAPI routes not registered")
+
+	// Create API client with bounded HTTP timeout to prevent indefinite hangs
+	// that were causing CI failures (goroutines blocking until 15-minute global timeout)
+	qService, err := queryapi.NewClientWithResponses(
+		apiContainers[QueryAPIName].URI,
+		queryapi.WithHTTPClient(GetHTTPClient()),
+	)
 	require.NoError(t, err)

 	return Network{
@@ -103,7 +103,7 @@ func CreateMockServer(t testing.TB) *MockServer {
 	require.NoError(t, err)

 	return &MockServer{
-		Client:    &http.Client{},
+		Client:    GetHTTPClient(),
 		Internal:  Address(fmt.Sprintf("http://%s:%d", mockServerAlias, port.Int())),
 		External:  Address(fmt.Sprintf("http://%s:%d", host, externalPort.Int())),
 		Container: container,
@@ -193,11 +193,12 @@ func WaitForMockEndpoint(t testing.TB, server *MockServer, request MockRequest)

 	verifyURL := fmt.Sprintf("%s/mockserver/verify", server.External)

-	clientTimeout := 500 * time.Millisecond
-	client := &http.Client{}
+	// Use configured HTTP client with timeout to prevent indefinite hangs
+	client := GetHTTPClient()

-	timeout := time.After(60 * time.Second)
-	ticker := time.NewTicker(clientTimeout)
+	// Use configurable polling timeout (longer in CI environments)
+	timeout := time.After(GetPollingTimeout())
+	ticker := time.NewTicker(GetPollInterval())
 	defer ticker.Stop()

 	slog.Info("Attempting to process request", "body", jsonData)
@@ -238,7 +239,8 @@ func retrieveMatchingRequest(t testing.TB, server *MockServer, request MockReque
 	req, err := http.NewRequest("PUT", retrieveURL, nil)
 	require.NoError(t, err)

-	client := &http.Client{}
+	// Use configured HTTP client with timeout to prevent indefinite hangs
+	client := GetHTTPClient()
 	resp, err := client.Do(req)
 	require.NoError(t, err)
 	defer resp.Body.Close()
@@ -88,8 +88,10 @@ func SetQueryForClient(t testing.TB, client queryapi.ClientWithResponsesInterfac
 func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.ClientWithResponsesInterface, id string, status queryapi.ClientStatus) {
 	t.Helper()

-	timeout := time.After(60 * time.Second)
-	ticker := time.NewTicker(500 * time.Millisecond) // Increased from 100ms to avoid rate limiting (2 req/s < 5 req/s limit)
+	// Use configurable polling timeout (longer in CI environments to handle slower infrastructure)
+	// and configurable poll interval to balance responsiveness with rate limiting concerns
+	timeout := time.After(test.GetPollingTimeout())
+	ticker := time.NewTicker(test.GetPollInterval())
 	defer ticker.Stop()

 	for {