Jay Brown
423 lines
13 KiB
Markdown
423 lines
13 KiB
Markdown
# Configuration Management
|
|
|
|
## Updated Configuration (September 2025)
|
|
|
|
The system has expanded configuration support for batch processing, enhanced observability, and Permit.io RBAC integration.
|
|
|
|
## Configuration Architecture
|
|
|
|
The system implements a hierarchical, environment-driven configuration system designed for flexibility, type safety, and testability. Configuration is managed through the `internal/serviceconfig` package with domain-specific sub-packages.
|
|
|
|
## Configuration Hierarchy
|
|
|
|
### Environment Variable Priority (First Wins)
|
|
1. **devbox.json "env"** section - Highest priority for development
|
|
2. **devbox.json "init_hook"** exports - Build-time configuration
|
|
3. **.env file** variables - Local overrides
|
|
4. **System environment** - Runtime environment variables
|
|
|
|
### Configuration Loading Process
|
|
```go
|
|
// 1. Load .env file if present (development)
|
|
godotenv.Load()
|
|
|
|
// 2. Parse environment variables into struct
|
|
env.Parse(&config)
|
|
|
|
// 3. Validate required fields
|
|
config.Validate()
|
|
|
|
// 4. Initialize dependent services
|
|
config.Initialize(ctx)
|
|
```
|
|
|
|
## Base Configuration Structure
|
|
|
|
### Core Configuration (`serviceconfig/common.go`)
|
|
```go
|
|
type BaseConfig struct {
|
|
// Application
|
|
LogLevel string `env:"LOG_LEVEL" envDefault:"INFO"`
|
|
|
|
// Server
|
|
Port int `env:"PORT" envDefault:"8080"`
|
|
HTTPHost string `env:"HTTP_HOST" envDefault:"0.0.0.0"`
|
|
|
|
// Feature Flags
|
|
DisableAuth bool `env:"DISABLE_AUTH" envDefault:"false"`
|
|
EnableOTEL bool `env:"ENABLE_OTEL" envDefault:"false"`
|
|
|
|
// Observability
|
|
MetricsPath string `env:"METRICS_PATH" envDefault:"/metrics"`
|
|
}
|
|
```
|
|
|
|
### Configuration Providers Pattern
|
|
```go
|
|
// Interface composition for service dependencies
|
|
type ConfigProvider interface {
|
|
GetLogLevel() string
|
|
GetPort() int
|
|
GetHTTPHost() string
|
|
IsAuthDisabled() bool
|
|
}
|
|
|
|
// Service-specific configuration embeds base
|
|
type ServiceConfig struct {
|
|
BaseConfig
|
|
DatabaseConfig database.Config
|
|
AWSConfig aws.Config
|
|
QueueConfig queue.Config
|
|
}
|
|
```
|
|
|
|
## Domain-Specific Configuration
|
|
|
|
### Database Configuration (`serviceconfig/database/`)
|
|
```go
|
|
type Config struct {
|
|
// Connection
|
|
Host string `env:"PGHOST" envDefault:"localhost"`
|
|
Port int `env:"PGPORT" envDefault:"5432"`
|
|
Database string `env:"PGDATABASE"`
|
|
Username string `env:"PGUSER"`
|
|
Password string `env:"PGPASSWORD"`
|
|
|
|
// Connection Pool
|
|
MaxOpenConns int `env:"DB_MAX_OPEN_CONNS" envDefault:"25"`
|
|
MaxIdleConns int `env:"DB_MAX_IDLE_CONNS" envDefault:"25"`
|
|
ConnMaxLifetime int `env:"DB_CONN_MAX_LIFETIME" envDefault:"300"`
|
|
|
|
// Security
|
|
SSLMode string `env:"DB_SSL_MODE" envDefault:"require"`
|
|
NoSSL bool `env:"DB_NOSSL" envDefault:"false"`
|
|
}
|
|
|
|
// Provider interface
|
|
type ConfigProvider interface {
|
|
GetDatabaseConfig() Config
|
|
}
|
|
|
|
// Connection pool management
|
|
func (c Config) CreatePool(ctx context.Context) (*pgxpool.Pool, error) {
|
|
dsn := c.buildConnectionString()
|
|
return pgxpool.New(ctx, dsn)
|
|
}
|
|
```
|
|
|
|
### AWS Configuration (`serviceconfig/aws/`)
|
|
```go
|
|
type Config struct {
|
|
// Credentials
|
|
Region string `env:"AWS_REGION" envDefault:"us-east-1"`
|
|
AccessKeyID string `env:"AWS_ACCESS_KEY_ID"`
|
|
SecretAccessKey string `env:"AWS_SECRET_ACCESS_KEY"`
|
|
SessionToken string `env:"AWS_SESSION_TOKEN"`
|
|
Profile string `env:"AWS_PROFILE"`
|
|
|
|
// Endpoints (for LocalStack/testing)
|
|
EndpointURL string `env:"AWS_ENDPOINT_URL"`
|
|
S3UsePathStyle bool `env:"AWS_S3_USE_PATH_STYLE" envDefault:"false"`
|
|
|
|
// Service-specific
|
|
S3Bucket string `env:"S3_BUCKET"`
|
|
TextractRoleARN string `env:"TEXTRACT_ROLE_ARN"`
|
|
}
|
|
|
|
// SDK integration
|
|
func (c Config) LoadAWSConfig(ctx context.Context) (aws.Config, error) {
|
|
var opts []func(*config.LoadOptions) error
|
|
|
|
if c.Region != "" {
|
|
opts = append(opts, config.WithRegion(c.Region))
|
|
}
|
|
|
|
if c.EndpointURL != "" {
|
|
opts = append(opts, config.WithEndpointResolverWithOptions(
|
|
aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
|
return aws.Endpoint{URL: c.EndpointURL}, nil
|
|
}),
|
|
))
|
|
}
|
|
|
|
return config.LoadDefaultConfig(ctx, opts...)
|
|
}
|
|
```
|
|
|
|
### Queue Configuration (`serviceconfig/queue/`)
|
|
```go
|
|
// Base queue configuration with batch processing support
|
|
type Config struct {
|
|
// Connection
|
|
QueueURL string `env:"QUEUE_URL"`
|
|
MaxMessages int32 `env:"QUEUE_MAX_MESSAGES" envDefault:"10"`
|
|
WaitTimeSeconds int32 `env:"QUEUE_WAIT_TIME" envDefault:"20"`
|
|
VisibilityTimeout int32 `env:"QUEUE_VISIBILITY_TIMEOUT" envDefault:"30"`
|
|
|
|
// Processing (enhanced for batch operations)
|
|
WorkerCount int `env:"QUEUE_WORKER_COUNT" envDefault:"5"`
|
|
MaxRetries int `env:"QUEUE_MAX_RETRIES" envDefault:"3"`
|
|
RetryDelay int `env:"QUEUE_RETRY_DELAY" envDefault:"5"`
|
|
BatchSize int `env:"QUEUE_BATCH_SIZE" envDefault:"10"`
|
|
}
|
|
|
|
// Service-specific queue configs
|
|
type DocumentInitConfig struct {
|
|
Config
|
|
DocumentSyncURL string `env:"DOCUMENT_SYNC_URL"`
|
|
}
|
|
|
|
type QueryConfig struct {
|
|
Config
|
|
QuerySyncURL string `env:"QUERY_SYNC_URL"`
|
|
}
|
|
```
|
|
|
|
### Authentication Configuration (`serviceconfig/auth/`)
|
|
```go
|
|
type Config struct {
|
|
// Cognito Configuration
|
|
UserPoolID string `env:"COGNITO_USER_POOL_ID"`
|
|
ClientID string `env:"COGNITO_CLIENT_ID"`
|
|
ClientSecret string `env:"COGNITO_CLIENT_SECRET" envSecret:"true"`
|
|
Domain string `env:"COGNITO_DOMAIN"`
|
|
|
|
// URLs
|
|
RedirectURL string `env:"COGNITO_REDIRECT_URL"`
|
|
LogoutURL string `env:"COGNITO_LOGOUT_URL"`
|
|
|
|
// JWT Configuration
|
|
JWKSCacheExpiry time.Duration `env:"JWKS_CACHE_EXPIRY" envDefault:"1h"`
|
|
TokenExpiry time.Duration `env:"TOKEN_EXPIRY" envDefault:"24h"`
|
|
|
|
// Feature Flags
|
|
DisableAuth bool `env:"DISABLE_AUTH" envDefault:"false"`
|
|
}
|
|
|
|
// Dynamic URL generation
|
|
func (c Config) GetAuthorizationURL() string {
|
|
return fmt.Sprintf("https://%s.auth.%s.amazoncognito.com/oauth2/authorize",
|
|
c.Domain, extractRegion(c.UserPoolID))
|
|
}
|
|
|
|
func (c Config) GetTokenURL() string {
|
|
return fmt.Sprintf("https://%s.auth.%s.amazoncognito.com/oauth2/token",
|
|
c.Domain, extractRegion(c.UserPoolID))
|
|
}
|
|
```
|
|
|
|
## Environment Variables Reference
|
|
|
|
### Core Application Variables
|
|
```bash
|
|
# Logging and Debugging
|
|
LOG_LEVEL=DEBUG|INFO|WARN|ERROR # Default: INFO
|
|
ENABLE_OTEL=true|false # Default: false
|
|
|
|
# Server Configuration
|
|
PORT=8080 # Default: 8080
|
|
HTTP_HOST=0.0.0.0 # Default: 0.0.0.0
|
|
METRICS_PATH=/metrics # Default: /metrics
|
|
|
|
# Feature Flags
|
|
DISABLE_AUTH=true|false # Default: false (enable for local dev)
|
|
```
|
|
|
|
### Database Variables
|
|
```bash
|
|
# Connection
|
|
PGHOST=localhost # Database host
|
|
PGPORT=5432 # Database port
|
|
PGDATABASE=queryorchestration # Database name
|
|
PGUSER=postgres # Database user
|
|
PGPASSWORD=password # Database password
|
|
|
|
# Connection Pool
|
|
DB_MAX_OPEN_CONNS=25 # Default: 25
|
|
DB_MAX_IDLE_CONNS=25 # Default: 25
|
|
DB_CONN_MAX_LIFETIME=300 # Seconds, Default: 300
|
|
|
|
# Security
|
|
DB_SSL_MODE=require|disable # Default: require
|
|
DB_NOSSL=true|false # Default: false (for local dev)
|
|
```
|
|
|
|
### AWS Configuration Variables
|
|
```bash
|
|
# Credentials
|
|
AWS_REGION=us-east-1 # AWS region
|
|
AWS_ACCESS_KEY_ID=your-access-key # AWS access key
|
|
AWS_SECRET_ACCESS_KEY=your-secret # AWS secret key
|
|
AWS_SESSION_TOKEN=session-token # Optional session token
|
|
AWS_PROFILE=default # Optional AWS profile
|
|
|
|
# LocalStack/Testing
|
|
AWS_ENDPOINT_URL=http://localstack:4566 # Override AWS endpoints
|
|
AWS_S3_USE_PATH_STYLE=true|false # S3 path style (LocalStack)
|
|
|
|
# Service Configuration
|
|
S3_BUCKET=documents-bucket # S3 bucket for documents
|
|
TEXTRACT_ROLE_ARN=arn:aws:iam::... # Textract service role
|
|
```
|
|
|
|
### Queue Configuration Variables
|
|
```bash
|
|
# Queue URLs (SQS)
|
|
STORE_EVENT_URL=https://sqs.region.amazonaws.com/account/store-events
|
|
DOCUMENT_INIT_URL=https://sqs.region.amazonaws.com/account/doc-init
|
|
DOCUMENT_SYNC_URL=https://sqs.region.amazonaws.com/account/doc-sync
|
|
DOCUMENT_CLEAN_URL=https://sqs.region.amazonaws.com/account/doc-clean
|
|
DOCUMENT_TEXT_URL=https://sqs.region.amazonaws.com/account/doc-text
|
|
QUERY_SYNC_URL=https://sqs.region.amazonaws.com/account/query-sync
|
|
QUERY_URL=https://sqs.region.amazonaws.com/account/query
|
|
CLIENT_SYNC_URL=https://sqs.region.amazonaws.com/account/client-sync
|
|
QUERY_VERSION_SYNC_URL=https://sqs.region.amazonaws.com/account/query-version-sync
|
|
|
|
# Queue Processing
|
|
QUEUE_MAX_MESSAGES=10 # Messages per batch
|
|
QUEUE_WAIT_TIME=20 # Long polling seconds
|
|
QUEUE_VISIBILITY_TIMEOUT=30 # Message visibility
|
|
QUEUE_WORKER_COUNT=5 # Concurrent workers
|
|
QUEUE_MAX_RETRIES=3 # Retry attempts
|
|
QUEUE_RETRY_DELAY=5 # Retry delay seconds
|
|
```
|
|
|
|
### Authentication Variables
|
|
```bash
|
|
# AWS Cognito
|
|
COGNITO_USER_POOL_ID=us-east-1_xxxxxxx # Cognito User Pool ID
|
|
COGNITO_CLIENT_ID=client-id # Cognito App Client ID
|
|
COGNITO_CLIENT_SECRET=client-secret # Cognito App Client Secret
|
|
COGNITO_DOMAIN=your-domain # Cognito domain name
|
|
|
|
# URLs
|
|
COGNITO_REDIRECT_URL=http://localhost:8080/login-callback
|
|
COGNITO_LOGOUT_URL=http://localhost:8080/logout
|
|
|
|
# JWT Settings
|
|
JWKS_CACHE_EXPIRY=1h # JWKS cache duration
|
|
TOKEN_EXPIRY=24h # JWT token expiry
|
|
```
|
|
|
|
### Authorization Variables (Permit.io RBAC)
|
|
```bash
|
|
# Permit.io API Configuration
|
|
PERMIT_IO_API_KEY=permit_key_xxxxxxx # Permit.io API key (project/env derived from key)
|
|
PERMIT_IO_TENANT=default # Permit.io tenant identifier (default: "default")
|
|
PERMIT_IO_PDP_URL=http://localhost:7766 # Policy Decision Point URL (optional, uses cloud if not set)
|
|
PERMIT_IO_BASE_URL=https://api.permit.io # Permit.io API base URL (optional, for testing)
|
|
```
|
|
|
|
**Note:** The Permit.io project ID and environment ID are automatically derived from the API key
|
|
using the `/v2/api-key/scope` endpoint. Use a project-level or environment-level API key;
|
|
organization-level API keys are not supported.
|
|
|
|
## Configuration Validation
|
|
|
|
### Required Field Validation
|
|
```go
|
|
type Config struct {
|
|
DatabaseURL string `env:"DATABASE_URL,required"`
|
|
S3Bucket string `env:"S3_BUCKET,required"`
|
|
QueueURL string `env:"QUEUE_URL,required"`
|
|
}
|
|
|
|
// Validation on startup
|
|
func (c *Config) Validate() error {
|
|
var errors []string
|
|
|
|
if c.DatabaseURL == "" {
|
|
errors = append(errors, "DATABASE_URL is required")
|
|
}
|
|
|
|
if c.S3Bucket == "" {
|
|
errors = append(errors, "S3_BUCKET is required")
|
|
}
|
|
|
|
if len(errors) > 0 {
|
|
return fmt.Errorf("configuration validation failed: %s", strings.Join(errors, ", "))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
```
|
|
|
|
### Secret Masking
|
|
```go
|
|
// Automatic masking of sensitive values in logs
|
|
type Config struct {
|
|
DatabasePassword string `env:"PGPASSWORD" envSecret:"true"`
|
|
AWSSecretKey string `env:"AWS_SECRET_ACCESS_KEY" envSecret:"true"`
|
|
JWTSecret string `env:"JWT_SECRET" envSecret:"true"`
|
|
}
|
|
|
|
// String() method automatically masks secrets
|
|
func (c Config) String() string {
|
|
return env.String(c) // Automatically masks fields tagged with envSecret
|
|
}
|
|
```
|
|
|
|
## Development vs Production Configuration
|
|
|
|
### Development Environment (.env)
|
|
```bash
|
|
# Local development overrides
|
|
LOG_LEVEL=DEBUG
|
|
DISABLE_AUTH=true
|
|
DB_NOSSL=true
|
|
AWS_ENDPOINT_URL=http://localstack:4566
|
|
AWS_S3_USE_PATH_STYLE=true
|
|
|
|
# Local service URLs
|
|
PGHOST=localhost
|
|
PGPORT=5432
|
|
```
|
|
|
|
### Production Environment
|
|
```bash
|
|
# Production settings
|
|
LOG_LEVEL=INFO
|
|
DISABLE_AUTH=false
|
|
DB_SSL_MODE=require
|
|
|
|
# Production AWS
|
|
AWS_REGION=us-east-1
|
|
# Credentials from IAM roles or environment
|
|
|
|
# Production database
|
|
PGHOST=production-db.region.rds.amazonaws.com
|
|
PGPORT=5432
|
|
DB_SSL_MODE=require
|
|
```
|
|
|
|
### Testing Environment
|
|
```bash
|
|
# Test-specific overrides
|
|
LOG_LEVEL=WARN
|
|
DISABLE_AUTH=true
|
|
DB_NOSSL=true
|
|
|
|
# Test containers
|
|
PGHOST=test-container
|
|
AWS_ENDPOINT_URL=http://test-localstack:4566
|
|
```
|
|
|
|
## Configuration Best Practices
|
|
|
|
### Security
|
|
- **Never commit secrets**: Use environment variables or secret management
|
|
- **Mask sensitive logs**: Use `envSecret` tag for automatic masking
|
|
- **Validate inputs**: Check configuration values at startup
|
|
- **Use least privilege**: Configure minimal required permissions
|
|
|
|
### Maintainability
|
|
- **Default values**: Provide sensible defaults for optional configuration
|
|
- **Documentation**: Document all environment variables and their purpose
|
|
- **Type safety**: Use strong typing and validation
|
|
- **Composition**: Build configuration from smaller, focused interfaces
|
|
|
|
### Testing
|
|
- **Override-friendly**: Allow easy configuration overrides for tests
|
|
- **Isolation**: Each test should have independent configuration
|
|
- **Realistic defaults**: Test configuration should mirror production patterns
|
|
- **Environment separation**: Clear separation between dev/test/prod configs |