query-orchestration/docs/ai.generated/07-configuration-management.md

Configuration Management

Updated Configuration (September 2025)

The system has expanded configuration support for batch processing, enhanced observability, and Permit.io RBAC integration.

Configuration Architecture

The system implements a hierarchical, environment-driven configuration system designed for flexibility, type safety, and testability. Configuration is managed through the internal/serviceconfig package with domain-specific sub-packages.

Configuration Hierarchy

Environment Variable Priority (First Wins)

1. devbox.json "env" section - Highest priority for development
2. devbox.json "init_hook" exports - Build-time configuration
3. .env file variables - Local overrides
4. System environment - Runtime environment variables

Configuration Loading Process

// 1. Load .env file if present (development)
godotenv.Load()

// 2. Parse environment variables into struct
env.Parse(&config)

// 3. Validate required fields
config.Validate()

// 4. Initialize dependent services
config.Initialize(ctx)

Base Configuration Structure

Core Configuration (serviceconfig/common.go)

type BaseConfig struct {
    // Application
    LogLevel string `env:"LOG_LEVEL" envDefault:"INFO"`
    
    // Server
    Port     int    `env:"PORT" envDefault:"8080"`
    HTTPHost string `env:"HTTP_HOST" envDefault:"0.0.0.0"`
    
    // Feature Flags
    DisableAuth bool `env:"DISABLE_AUTH" envDefault:"false"`
    EnableOTEL  bool `env:"ENABLE_OTEL" envDefault:"false"`
    
    // Observability
    MetricsPath string `env:"METRICS_PATH" envDefault:"/metrics"`
}

Configuration Providers Pattern

// Interface composition for service dependencies
type ConfigProvider interface {
    GetLogLevel() string
    GetPort() int
    GetHTTPHost() string
    IsAuthDisabled() bool
}

// Service-specific configuration embeds base
type ServiceConfig struct {
    BaseConfig
    DatabaseConfig    database.Config
    AWSConfig        aws.Config
    QueueConfig      queue.Config
}

Domain-Specific Configuration

Database Configuration (serviceconfig/database/)

type Config struct {
    // Connection
    Host     string `env:"PGHOST" envDefault:"localhost"`
    Port     int    `env:"PGPORT" envDefault:"5432"`
    Database string `env:"PGDATABASE"`
    Username string `env:"PGUSER"`
    Password string `env:"PGPASSWORD"`
    
    // Connection Pool
    MaxOpenConns    int `env:"DB_MAX_OPEN_CONNS" envDefault:"25"`
    MaxIdleConns    int `env:"DB_MAX_IDLE_CONNS" envDefault:"25"`
    ConnMaxLifetime int `env:"DB_CONN_MAX_LIFETIME" envDefault:"300"`
    
    // Security
    SSLMode string `env:"DB_SSL_MODE" envDefault:"require"`
    NoSSL   bool   `env:"DB_NOSSL" envDefault:"false"`
}

// Provider interface
type ConfigProvider interface {
    GetDatabaseConfig() Config
}

// Connection pool management
func (c Config) CreatePool(ctx context.Context) (*pgxpool.Pool, error) {
    dsn := c.buildConnectionString()
    return pgxpool.New(ctx, dsn)
}

AWS Configuration (serviceconfig/aws/)

type Config struct {
    // Credentials
    Region          string `env:"AWS_REGION" envDefault:"us-east-1"`
    AccessKeyID     string `env:"AWS_ACCESS_KEY_ID"`
    SecretAccessKey string `env:"AWS_SECRET_ACCESS_KEY"`
    SessionToken    string `env:"AWS_SESSION_TOKEN"`
    Profile         string `env:"AWS_PROFILE"`
    
    // Endpoints (for LocalStack/testing)
    EndpointURL       string `env:"AWS_ENDPOINT_URL"`
    S3UsePathStyle    bool   `env:"AWS_S3_USE_PATH_STYLE" envDefault:"false"`
    
    // Service-specific
    S3Bucket        string `env:"S3_BUCKET"`
    TextractRoleARN string `env:"TEXTRACT_ROLE_ARN"`
}

// SDK integration
func (c Config) LoadAWSConfig(ctx context.Context) (aws.Config, error) {
    var opts []func(*config.LoadOptions) error
    
    if c.Region != "" {
        opts = append(opts, config.WithRegion(c.Region))
    }
    
    if c.EndpointURL != "" {
        opts = append(opts, config.WithEndpointResolverWithOptions(
            aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
                return aws.Endpoint{URL: c.EndpointURL}, nil
            }),
        ))
    }
    
    return config.LoadDefaultConfig(ctx, opts...)
}

Queue Configuration (serviceconfig/queue/)

// Base queue configuration with batch processing support
type Config struct {
    // Connection
    QueueURL          string `env:"QUEUE_URL"`
    MaxMessages       int32  `env:"QUEUE_MAX_MESSAGES" envDefault:"10"`
    WaitTimeSeconds   int32  `env:"QUEUE_WAIT_TIME" envDefault:"20"`
    VisibilityTimeout int32  `env:"QUEUE_VISIBILITY_TIMEOUT" envDefault:"30"`
    
    // Processing (enhanced for batch operations)
    WorkerCount     int `env:"QUEUE_WORKER_COUNT" envDefault:"5"`
    MaxRetries      int `env:"QUEUE_MAX_RETRIES" envDefault:"3"`
    RetryDelay      int `env:"QUEUE_RETRY_DELAY" envDefault:"5"`
    BatchSize       int `env:"QUEUE_BATCH_SIZE" envDefault:"10"`
}

// Service-specific queue configs
type DocumentInitConfig struct {
    Config
    DocumentSyncURL string `env:"DOCUMENT_SYNC_URL"`
}

type QueryConfig struct {
    Config
    QuerySyncURL string `env:"QUERY_SYNC_URL"`
}

Authentication Configuration (serviceconfig/auth/)

type Config struct {
    // Cognito Configuration
    UserPoolID     string `env:"COGNITO_USER_POOL_ID"`
    ClientID       string `env:"COGNITO_CLIENT_ID"`
    ClientSecret   string `env:"COGNITO_CLIENT_SECRET" envSecret:"true"`
    Domain         string `env:"COGNITO_DOMAIN"`
    
    // URLs
    RedirectURL    string `env:"COGNITO_REDIRECT_URL"`
    LogoutURL      string `env:"COGNITO_LOGOUT_URL"`
    
    // JWT Configuration
    JWKSCacheExpiry time.Duration `env:"JWKS_CACHE_EXPIRY" envDefault:"1h"`
    TokenExpiry     time.Duration `env:"TOKEN_EXPIRY" envDefault:"24h"`
    
    // Feature Flags
    DisableAuth bool `env:"DISABLE_AUTH" envDefault:"false"`
}

// Dynamic URL generation
func (c Config) GetAuthorizationURL() string {
    return fmt.Sprintf("https://%s.auth.%s.amazoncognito.com/oauth2/authorize", 
        c.Domain, extractRegion(c.UserPoolID))
}

func (c Config) GetTokenURL() string {
    return fmt.Sprintf("https://%s.auth.%s.amazoncognito.com/oauth2/token",
        c.Domain, extractRegion(c.UserPoolID))
}

Environment Variables Reference

Core Application Variables

# Logging and Debugging
LOG_LEVEL=DEBUG|INFO|WARN|ERROR         # Default: INFO
ENABLE_OTEL=true|false                  # Default: false

# Server Configuration  
PORT=8080                               # Default: 8080
HTTP_HOST=0.0.0.0                      # Default: 0.0.0.0
METRICS_PATH=/metrics                   # Default: /metrics

# Feature Flags
DISABLE_AUTH=true|false                 # Default: false (enable for local dev)

Database Variables

# Connection
PGHOST=localhost                        # Database host
PGPORT=5432                            # Database port
PGDATABASE=queryorchestration          # Database name
PGUSER=postgres                        # Database user
PGPASSWORD=password                    # Database password

# Connection Pool
DB_MAX_OPEN_CONNS=25                   # Default: 25
DB_MAX_IDLE_CONNS=25                   # Default: 25
DB_CONN_MAX_LIFETIME=300               # Seconds, Default: 300

# Security
DB_SSL_MODE=require|disable            # Default: require
DB_NOSSL=true|false                    # Default: false (for local dev)

AWS Configuration Variables

# Credentials
AWS_REGION=us-east-1                   # AWS region
AWS_ACCESS_KEY_ID=your-access-key      # AWS access key
AWS_SECRET_ACCESS_KEY=your-secret      # AWS secret key
AWS_SESSION_TOKEN=session-token        # Optional session token
AWS_PROFILE=default                    # Optional AWS profile

# LocalStack/Testing
AWS_ENDPOINT_URL=http://localstack:4566 # Override AWS endpoints
AWS_S3_USE_PATH_STYLE=true|false       # S3 path style (LocalStack)

# Service Configuration
S3_BUCKET=documents-bucket             # S3 bucket for documents
TEXTRACT_ROLE_ARN=arn:aws:iam::...     # Textract service role

Queue Configuration Variables

# Queue URLs (SQS)
STORE_EVENT_URL=https://sqs.region.amazonaws.com/account/store-events
DOCUMENT_INIT_URL=https://sqs.region.amazonaws.com/account/doc-init
DOCUMENT_SYNC_URL=https://sqs.region.amazonaws.com/account/doc-sync
DOCUMENT_CLEAN_URL=https://sqs.region.amazonaws.com/account/doc-clean
DOCUMENT_TEXT_URL=https://sqs.region.amazonaws.com/account/doc-text
QUERY_SYNC_URL=https://sqs.region.amazonaws.com/account/query-sync
QUERY_URL=https://sqs.region.amazonaws.com/account/query
CLIENT_SYNC_URL=https://sqs.region.amazonaws.com/account/client-sync
QUERY_VERSION_SYNC_URL=https://sqs.region.amazonaws.com/account/query-version-sync

# Queue Processing
QUEUE_MAX_MESSAGES=10                  # Messages per batch
QUEUE_WAIT_TIME=20                     # Long polling seconds
QUEUE_VISIBILITY_TIMEOUT=30            # Message visibility
QUEUE_WORKER_COUNT=5                   # Concurrent workers
QUEUE_MAX_RETRIES=3                    # Retry attempts
QUEUE_RETRY_DELAY=5                    # Retry delay seconds

Authentication Variables

# AWS Cognito
COGNITO_USER_POOL_ID=us-east-1_xxxxxxx # Cognito User Pool ID
COGNITO_CLIENT_ID=client-id            # Cognito App Client ID
COGNITO_CLIENT_SECRET=client-secret    # Cognito App Client Secret
COGNITO_DOMAIN=your-domain             # Cognito domain name

# URLs
COGNITO_REDIRECT_URL=http://localhost:8080/login-callback
COGNITO_LOGOUT_URL=http://localhost:8080/logout

# JWT Settings
JWKS_CACHE_EXPIRY=1h                   # JWKS cache duration
TOKEN_EXPIRY=24h                       # JWT token expiry

Configuration Validation

Required Field Validation

type Config struct {
    DatabaseURL string `env:"DATABASE_URL,required"`
    S3Bucket    string `env:"S3_BUCKET,required"`
    QueueURL    string `env:"QUEUE_URL,required"`
}

// Validation on startup
func (c *Config) Validate() error {
    var errors []string
    
    if c.DatabaseURL == "" {
        errors = append(errors, "DATABASE_URL is required")
    }
    
    if c.S3Bucket == "" {
        errors = append(errors, "S3_BUCKET is required")
    }
    
    if len(errors) > 0 {
        return fmt.Errorf("configuration validation failed: %s", strings.Join(errors, ", "))
    }
    
    return nil
}

Secret Masking

// Automatic masking of sensitive values in logs
type Config struct {
    DatabasePassword string `env:"PGPASSWORD" envSecret:"true"`
    AWSSecretKey     string `env:"AWS_SECRET_ACCESS_KEY" envSecret:"true"`
    JWTSecret        string `env:"JWT_SECRET" envSecret:"true"`
}

// String() method automatically masks secrets
func (c Config) String() string {
    return env.String(c) // Automatically masks fields tagged with envSecret
}

Development vs Production Configuration

Development Environment (.env)

# Local development overrides
LOG_LEVEL=DEBUG
DISABLE_AUTH=true
DB_NOSSL=true
AWS_ENDPOINT_URL=http://localstack:4566
AWS_S3_USE_PATH_STYLE=true

# Local service URLs
PGHOST=localhost
PGPORT=5432

Production Environment

# Production settings
LOG_LEVEL=INFO
DISABLE_AUTH=false
DB_SSL_MODE=require

# Production AWS
AWS_REGION=us-east-1
# Credentials from IAM roles or environment

# Production database
PGHOST=production-db.region.rds.amazonaws.com
PGPORT=5432
DB_SSL_MODE=require

Testing Environment

# Test-specific overrides
LOG_LEVEL=WARN
DISABLE_AUTH=true
DB_NOSSL=true

# Test containers
PGHOST=test-container
AWS_ENDPOINT_URL=http://test-localstack:4566

Configuration Best Practices

Security

• Never commit secrets: Use environment variables or secret management
• Mask sensitive logs: Use envSecret tag for automatic masking
• Validate inputs: Check configuration values at startup
• Use least privilege: Configure minimal required permissions

Maintainability

• Default values: Provide sensible defaults for optional configuration
• Documentation: Document all environment variables and their purpose
• Type safety: Use strong typing and validation
• Composition: Build configuration from smaller, focused interfaces

Testing

• Override-friendly: Allow easy configuration overrides for tests
• Isolation: Each test should have independent configuration
• Realistic defaults: Test configuration should mirror production patterns
• Environment separation: Clear separation between dev/test/prod configs