Merged in feature/common-metrics (pull request #73)
Add common metrics * add common metrics and sample for using * updates * attempt integration * wip * working stubs generate * docs * comments * Merge branch 'main' of bitbucket.org:aarete/query-orchestration into feature/common-metrics * runner metrics all * exclude generated from the coverage * fix lint * refactor metrics * change .gen filename * new gen * Merge branch 'main' of bitbucket.org:aarete/query-orchestration into feature/common-metrics * docs * main merge * refactor * add to generate * merge main
This commit is contained in:
Jay Brown
@@ -37,5 +37,21 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// optional add shutdown metrics
|
||||
defer func() {
|
||||
err = cfg.GetMetrics().RecordLifecycleEvent(1.0, "shutdown", "main")
|
||||
|
||||
if err != nil {
|
||||
slog.Error("Failed to record shutdown metric ", "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// start and block until server is done
|
||||
server.Listen(ctx)
|
||||
|
||||
// optional shutdown
|
||||
metrics := cfg.GetMetrics()
|
||||
if err := metrics.Shutdown(ctx); err != nil {
|
||||
slog.Error("Error shutting down metrics server", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
+110
-77
@@ -27,99 +27,73 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Create a context that we can cancel
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Define metrics configuration
|
||||
// This example shows all supported metric types and their configurations
|
||||
standardDefinitions := prometheus.NewStandardMetricDefinitions()
|
||||
// Define metrics configuration using StandardMetricDefinitions
|
||||
config := prometheus.MetricsConfig{
|
||||
Port: 8080,
|
||||
Namespace: "myapp",
|
||||
Subsystem: "api",
|
||||
// When ServeHttp is true, the package will start its own HTTP server
|
||||
// If you want to use your own HTTP server (e.g., with Echo middleware),
|
||||
// set this to false and configure your server to handle the /metrics endpoint
|
||||
ServeHttp: true,
|
||||
Definitions: []prometheus.MetricDefinition{
|
||||
{
|
||||
// Counter: Only increases, never decreases
|
||||
Type: prometheus.Counter,
|
||||
Name: "api_requests_total",
|
||||
Help: "Total number of API requests",
|
||||
Labels: []string{"method", "endpoint", "status"},
|
||||
},
|
||||
{
|
||||
// Gauge: Can increase and decrease
|
||||
Type: prometheus.Gauge,
|
||||
Name: "active_users",
|
||||
Help: "Number of currently active users",
|
||||
Labels: []string{"user_type"},
|
||||
},
|
||||
{
|
||||
// Histogram: Tracks distribution of values
|
||||
Type: prometheus.Histogram,
|
||||
Name: "request_duration_seconds",
|
||||
Help: "Request duration distribution",
|
||||
Labels: []string{"endpoint"},
|
||||
Buckets: []float64{0.1, 0.5, 1, 2, 5}, // Define your bucket boundaries
|
||||
},
|
||||
{
|
||||
// Summary: Calculates quantiles over a sliding time window
|
||||
Type: prometheus.Summary,
|
||||
Name: "response_size_bytes",
|
||||
Help: "Response size distribution",
|
||||
Labels: []string{"endpoint"},
|
||||
Objectives: map[float64]float64{
|
||||
0.5: 0.05, // 50th percentile with 5% error
|
||||
0.9: 0.01, // 90th percentile with 1% error
|
||||
0.99: 0.001, // 99th percentile with 0.1% error
|
||||
// Start with standard metrics and append our custom one
|
||||
Definitions: append(standardDefinitions,
|
||||
prometheus.MetricDefinition{
|
||||
Type: prometheus.Counter,
|
||||
Name: "test_custom_events_total",
|
||||
Help: "Total number of custom test events",
|
||||
Labels: []string{
|
||||
"event_type",
|
||||
"priority",
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
}
|
||||
|
||||
// Create new metrics instance with context
|
||||
metrics, err := prometheus.New(ctx, config)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create metrics: %v", err)
|
||||
}
|
||||
|
||||
// Start a goroutine to simulate metric recording
|
||||
go simulateMetrics(ctx, metrics)
|
||||
|
||||
// Set up graceful shutdown
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
fmt.Println("Running... Press Ctrl+C to exit")
|
||||
|
||||
// Wait for interrupt signal
|
||||
<-sigChan
|
||||
fmt.Println("\nShutting down...")
|
||||
|
||||
// Cancel context to stop metric simulation
|
||||
cancel()
|
||||
|
||||
// Create a timeout context for shutdown
|
||||
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer shutdownCancel()
|
||||
|
||||
// Gracefully shutdown the metrics server
|
||||
if err := metrics.Shutdown(shutdownCtx); err != nil {
|
||||
log.Printf("Error shutting down metrics server: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// simulateMetrics demonstrates how to record different types of metrics
|
||||
// in a concurrent environment. It takes a context for cancellation and
|
||||
// the metrics instance.
|
||||
func simulateMetrics(ctx context.Context, metrics *prometheus.Metrics) {
|
||||
// Sample data for simulation
|
||||
endpoints := []string{"/api/users", "/api/products", "/api/orders"}
|
||||
methods := []string{"GET", "POST", "PUT", "DELETE"}
|
||||
statuses := []string{"200", "400", "500"}
|
||||
userTypes := []string{"free", "premium", "enterprise"}
|
||||
errorTypes := []string{"none", "validation", "database", "timeout"}
|
||||
dependencies := []string{"database", "cache", "auth-service", "email-service"}
|
||||
cacheOperations := []string{"hit", "miss", "set", "eviction"}
|
||||
taskTypes := []string{"cleanup", "indexing", "backup", "sync"}
|
||||
taskStatuses := []string{"running", "queued", "failed"}
|
||||
results := []string{"success", "failure", "timeout"}
|
||||
components := []string{"api", "database", "cache", "auth"}
|
||||
queueNames := []string{"main-queue", "background-queue", "notification-queue"}
|
||||
resourceTypes := []string{"cpu", "memory", "disk", "network"}
|
||||
|
||||
// Custom metric sample data
|
||||
eventTypes := []string{"scheduled", "triggered", "manual"}
|
||||
priorities := []string{"low", "medium", "high"}
|
||||
|
||||
ticker := time.NewTicker(2 * time.Second)
|
||||
defer ticker.Stop()
|
||||
@@ -129,46 +103,105 @@ func simulateMetrics(ctx context.Context, metrics *prometheus.Metrics) {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
// Simulate request counter (Counter type)
|
||||
// Simulate standard metrics
|
||||
|
||||
// 1. Requests total
|
||||
endpoint := endpoints[rand.Intn(len(endpoints))]
|
||||
method := methods[rand.Intn(len(methods))]
|
||||
status := statuses[rand.Intn(len(statuses))]
|
||||
errorType := errorTypes[rand.Intn(len(errorTypes))]
|
||||
|
||||
err := metrics.RecordMetric("api_requests_total", 1, map[string]string{
|
||||
"method": method,
|
||||
"endpoint": endpoint,
|
||||
"status": status,
|
||||
})
|
||||
err := metrics.RecordRequestsTotal(1.0, endpoint, method, endpoint, status, errorType)
|
||||
if err != nil {
|
||||
log.Printf("Error recording request metric: %v", err)
|
||||
log.Printf("Error recording requests metric: %v", err)
|
||||
}
|
||||
|
||||
// Simulate active users (Gauge type)
|
||||
userType := userTypes[rand.Intn(len(userTypes))]
|
||||
activeUsers := rand.Float64() * 100
|
||||
err = metrics.RecordMetric("active_users", activeUsers, map[string]string{
|
||||
"user_type": userType,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Error recording active users metric: %v", err)
|
||||
}
|
||||
|
||||
// Simulate request duration (Histogram type)
|
||||
// 2. Request duration
|
||||
duration := rand.Float64() * 3 // Random duration between 0 and 3 seconds
|
||||
err = metrics.RecordMetric("request_duration_seconds", duration, map[string]string{
|
||||
"endpoint": endpoint,
|
||||
})
|
||||
err = metrics.RecordRequestDurationSeconds(duration, endpoint, method, endpoint, status)
|
||||
if err != nil {
|
||||
log.Printf("Error recording duration metric: %v", err)
|
||||
}
|
||||
|
||||
// Simulate response size (Summary type)
|
||||
responseSize := rand.Float64() * 5000 // Random size between 0 and 5000 bytes
|
||||
err = metrics.RecordMetric("response_size_bytes", responseSize, map[string]string{
|
||||
"endpoint": endpoint,
|
||||
// 3. Active operations
|
||||
activeOps := rand.Float64() * 50
|
||||
err = metrics.RecordActiveOperations(activeOps, method)
|
||||
if err != nil {
|
||||
log.Printf("Error recording active operations metric: %v", err)
|
||||
}
|
||||
|
||||
// 4. Resource usage
|
||||
resourceType := resourceTypes[rand.Intn(len(resourceTypes))]
|
||||
err = metrics.RecordResourceUsage(rand.Float64()*100, resourceType, "main-instance")
|
||||
if err != nil {
|
||||
log.Printf("Error recording resource usage metric: %v", err)
|
||||
}
|
||||
|
||||
// 5. Dependency health
|
||||
dep := dependencies[rand.Intn(len(dependencies))]
|
||||
health := rand.Float64() // Random health score between 0 and 1
|
||||
err = metrics.RecordDependencyHealth(health, dep, "external_service")
|
||||
if err != nil {
|
||||
log.Printf("Error recording dependency health metric: %v", err)
|
||||
}
|
||||
|
||||
// 6. Cache operations
|
||||
cacheOp := cacheOperations[rand.Intn(len(cacheOperations))]
|
||||
result := results[rand.Intn(len(results))]
|
||||
err = metrics.RecordCacheOperationsTotal(1.0, cacheOp, "main-cache", result)
|
||||
if err != nil {
|
||||
log.Printf("Error recording cache operations metric: %v", err)
|
||||
}
|
||||
|
||||
// 7. Cache operation duration
|
||||
err = metrics.RecordCacheOperationDurationSeconds(rand.Float64()*0.1, cacheOp, result)
|
||||
if err != nil {
|
||||
log.Printf("Error recording cache duration metric: %v", err)
|
||||
}
|
||||
|
||||
// 8. Background tasks
|
||||
taskType := taskTypes[rand.Intn(len(taskTypes))]
|
||||
taskStatus := taskStatuses[rand.Intn(len(taskStatuses))]
|
||||
err = metrics.RecordBackgroundTasks(rand.Float64()*10, taskType, taskStatus)
|
||||
if err != nil {
|
||||
log.Printf("Error recording background tasks metric: %v", err)
|
||||
}
|
||||
|
||||
// 9. Active connections
|
||||
err = metrics.RecordActiveConnections(rand.Float64()*100, "http")
|
||||
if err != nil {
|
||||
log.Printf("Error recording active connections metric: %v", err)
|
||||
}
|
||||
|
||||
// 10. Queue length
|
||||
queueName := queueNames[rand.Intn(len(queueNames))]
|
||||
err = metrics.RecordQueueLength(rand.Float64()*50, queueName)
|
||||
if err != nil {
|
||||
log.Printf("Error recording queue length metric: %v", err)
|
||||
}
|
||||
|
||||
// 11. Errors total
|
||||
component := components[rand.Intn(len(components))]
|
||||
err = metrics.RecordErrorsTotal(1.0, component, errorType)
|
||||
if err != nil {
|
||||
log.Printf("Error recording errors total metric: %v", err)
|
||||
}
|
||||
|
||||
// 12. Items in progress
|
||||
err = metrics.RecordItemsInProgress(rand.Float64()*20, endpoint, component)
|
||||
if err != nil {
|
||||
log.Printf("Error recording items in progress metric: %v", err)
|
||||
}
|
||||
|
||||
// Custom metric (still using RecordMetric since it's not part of the generated stubs)
|
||||
eventType := eventTypes[rand.Intn(len(eventTypes))]
|
||||
priority := priorities[rand.Intn(len(priorities))]
|
||||
err = metrics.RecordMetric("test_custom_events_total", 1, map[string]string{
|
||||
"event_type": eventType,
|
||||
"priority": priority,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Error recording response size metric: %v", err)
|
||||
log.Printf("Error recording custom event metric: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user