jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/prometheus (pull request #52)

Browse Source 
API instrumentation - metrics and logging

* api instrumentation

prometheus metrics and logging for all routes

* Merge branch 'main' of bitbucket.org:aarete/query-orchestration into feature/prometheus

* add log_level

and sync the config initialize

* docs

* cleanup

* add to compose:up:test

* docs for prometheus testing

* custom metrics

* cleanup

* merge main

* add tests

* cleanup docs

* cleanup

* lint

* fix unit tests (attempt)

* fix tests

* resolvesetlogger

* expose 8080

* compose changes

* bugfixesformichael

* don't build _test

* merge in main

* job_sync_runner
This commit is contained in:
Jay Brown
2025-02-13 15:44:08 +00:00
parent f5bd522faf
commit 477518e5eb
203 changed files with 36848 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/server/service/listener.go
+35 -1
View File
@@ -8,8 +8,10 @@ import (
"net/http"
"queryorchestration/internal/server"
"strconv"
"time"
"github.com/getkin/kin-openapi/openapi3"
"github.com/labstack/echo-contrib/echoprometheus"
"github.com/labstack/echo/v4"
"github.com/labstack/echo/v4/middleware"
_ "github.com/lib/pq"
@@ -59,6 +61,33 @@ type Server struct {
cleanup func() error
}
func getSlogCustomEchoMiddleware(logger *slog.Logger) echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
start := time.Now()
err := next(c)
if err != nil {
c.Error(err)
}
req := c.Request()
res := c.Response()
logger.Info("HTTP request",
"method", req.Method,
"uri", req.RequestURI,
"status", res.Status,
"latency", time.Since(start).String(),
"error", err,
"remote_ip", c.RealIP(),
)
return err
}
}
}
func New(ctx context.Context, cfg Config) (*Server, error) {
cleanup, err := server.New(ctx, cfg)
@@ -68,7 +97,12 @@ func New(ctx context.Context, cfg Config) (*Server, error) {
e := echo.New()
e.Use(middleware.Logger())
// Add Prometheus middleware
e.Use(echoprometheus.NewMiddleware("echo")) // Add the middleware
e.GET("/metrics", echoprometheus.NewHandler())
// Other middlewares
e.Use(getSlogCustomEchoMiddleware(cfg.GetLogger()))
e.Use(middleware.Recover())
e.Use(middleware.CORS())
internal/serviceconfig/common.go
+33 -6
View File
@@ -11,6 +11,7 @@ import (
"queryorchestration/internal/serviceconfig/queue"
"reflect"
"strings"
"sync"
"github.com/caarlos0/env/v11"
"github.com/joho/godotenv"
@@ -90,13 +91,38 @@ func getBaseConfig(cfg ConfigProvider) *BaseConfig {
return baseConfig
}
var (
initOnce sync.Once
initErr error
)
func ResetConfigInitOnceTestOnly() {
initOnce = sync.Once{}
initErr = nil
}
// InitializeConfig will initialize all of the configuration for the serivice
// and is guaranteed to run only once however many times it gets called.
func InitializeConfig(cfg ConfigProvider) error {
var initErr error
initOnce.Do(func() {
initErr = initializeConfigPrivate(cfg)
})
return initErr
}
// initializeConfigPrivate is the internal implementation of InitializeConfig so that
// testing can call it multiple times.
func initializeConfigPrivate(cfg ConfigProvider) error {
baseConfig := getBaseConfig(cfg)
if baseConfig == nil {
return errors.New("no BaseConfig found in the provided config struct")
initErr = errors.New("no BaseConfig found in the provided config struct")
return initErr
}
baseConfig.SetDefaultLogger()
// Initialize with info level
cfg.SetDefaultLogger()
if err := godotenv.Load(); err != nil {
slog.Warn("No .env file found or error loading it", "error", err)
@@ -111,13 +137,14 @@ func InitializeConfig(cfg ConfigProvider) error {
} else {
slog.Error("Error parsing environment variables", "error", err)
}
return err
initErr = err
return initErr
}
baseConfig.SetDefaultLogger()
// Reinitialize with environment level
cfg.SetDefaultLogger()
return nil
return initErr
}
func (b *BaseConfig) GetBasePath() string {
internal/serviceconfig/common_test.go
+12 -67
View File
@@ -1,8 +1,6 @@
package serviceconfig
import (
"bytes"
"log/slog"
"os"
"queryorchestration/internal/serviceconfig/aws"
"queryorchestration/internal/serviceconfig/database"
@@ -12,7 +10,8 @@ import (
"github.com/stretchr/testify/assert"
)
type TestingServiceNameConfig struct {
// Struct specifically for initialize config testing
type initializeConfigTestStruct struct {
BaseConfig
AppEnv string `env:"APP_ENV"`
BoolTest bool `env:"BOOL_TEST"`
@@ -93,14 +92,17 @@ func TestInitializeConfig(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Clear environment at the start of each subtest
os.Clearenv()
// Set environment variables for this test case
for k, v := range tt.envVars {
t.Setenv(k, v)
}
cfg := &TestingServiceNameConfig{}
err := InitializeConfig(cfg)
// Use the test-specific config struct
cfg := &initializeConfigTestStruct{}
err := initializeConfigPrivate(cfg)
logger := cfg.GetLogger()
assert.NotNil(t, logger)
@@ -111,11 +113,10 @@ func TestInitializeConfig(t *testing.T) {
t.Errorf("InitializeConfig() error = nil, wantErr %v", tt.wantErr)
return
}
// if we expect an error, check if the error message tt.errMessageContains is contained in the error message
// Check error message contains expected string
if tt.errMessageContains != "" && !strings.Contains(err.Error(), tt.errMessageContains) {
t.Errorf("InitializeConfig() error = %v, want error containing %v", err, tt.errMessageContains)
}
} else {
if err != nil {
t.Errorf("InitializeConfig() unexpected error = %v", err)
@@ -136,12 +137,15 @@ func TestInitializeConfig(t *testing.T) {
}
})
}
os.Clearenv()
}
func TestGetBaseConfig(t *testing.T) {
assert.NotNil(t, getBaseConfig(&BaseConfig{}))
assert.Nil(t, getBaseConfig(nil))
assert.NotNil(t, getBaseConfig(&TestingServiceNameConfig{}))
assert.NotNil(t, getBaseConfig(&initializeConfigTestStruct{}))
os.Clearenv()
}
func TestGetBasePath(t *testing.T) {
@@ -176,62 +180,3 @@ func TestSetAWSConfig(t *testing.T) {
cfg.SetAWSConfig(&acfg)
assert.Equal(t, acfg, cfg.AWSConfig)
}
func TestPrintConfigRecursive(t *testing.T) {
envVars := map[string]string{
"DB_USER": "postgres",
"DB_PASS": "password",
"DB_HOST": "localhost",
"DB_PORT": "5432",
"DB_NAME": "query_orchestration",
"DB_NOSSL": "true",
"AWS_ACCESS_KEY_ID": "key",
"AWS_SECRET_ACCESS_KEY": "password",
"AWS_REGION": "region",
"PWD": "/foo",
}
// Set environment variables
for k, v := range envVars {
t.Setenv(k, v)
}
// Create a buffer to capture log output
var logBuffer bytes.Buffer
// Create a test logger that writes to our buffer
testLogger := slog.New(slog.NewTextHandler(&logBuffer, &slog.HandlerOptions{
Level: slog.LevelInfo,
}))
cfg := &BaseConfig{}
err := InitializeConfig(cfg)
assert.NoError(t, err)
cfg.Logger = testLogger
// Call the function we're testing
cfg.PrintConfig("secret")
// Get the log output as a string
logOutput := logBuffer.String()
// Check that each environment variable value appears in the logs
// except for DB_PASS which should be masked
for envKey, envValue := range envVars {
if envKey == "DB_PASS" || envKey == "AWS_SECRET_ACCESS_KEY" {
// Password should be masked
if !strings.Contains(logOutput, "pas") {
t.Errorf("Expected masked password in logs, got: %s", logOutput)
}
// Full password should not appear
if strings.Contains(logOutput, envValue) {
t.Errorf("Full password should not appear in logs: %s", logOutput)
}
} else {
if !strings.Contains(logOutput, envValue) {
t.Errorf("Expected %s in logs, but it was not found. Log output: %s", envValue, logOutput)
}
}
}
}
internal/serviceconfig/commonlog_test.go
+74
View File
@@ -0,0 +1,74 @@
package serviceconfig_test
import (
"bytes"
"log/slog"
"os"
"queryorchestration/internal/serviceconfig"
"strings"
"testing"
)
// TestPrintConfigRecursive tests the PrintConfigRecursive function as a blackbox test.
func TestPrintConfigRecursive(t *testing.T) {
serviceconfig.ResetConfigInitOnceTestOnly()
os.Clearenv()
envVars := map[string]string{
"DB_USER": "postgres",
"DB_PASS": "password",
"DB_HOST": "localhost",
"DB_PORT": "5432",
"DB_NAME": "query_orchestration",
"DB_NOSSL": "true",
"AWS_ACCESS_KEY_ID": "key",
"AWS_SECRET_ACCESS_KEY": "password",
"AWS_REGION": "region",
"PWD": "/foo", // Only include vars needed by BaseConfig
}
// Set environment variables
for k, v := range envVars {
t.Setenv(k, v)
}
// Create a buffer to capture log output
var logBuffer bytes.Buffer
// Create a test logger that writes to our buffer
testLogger := slog.New(slog.NewTextHandler(&logBuffer, &slog.HandlerOptions{
Level: slog.LevelDebug,
}))
// Use BaseConfig instead of initializeConfigTestStruct
cfg := &serviceconfig.BaseConfig{}
err := serviceconfig.InitializeConfig(cfg)
if err != nil {
t.Fatalf("Error initializing config: %v", err)
}
// Set logger after initialization
cfg.Logger = testLogger
// Call the function we're testing
cfg.PrintConfig("secret")
// Get the log output as a string
logOutput := logBuffer.String()
// Check that each environment variable value appears in the logs
// except for sensitive values which should be masked
for envKey, envValue := range envVars {
if envKey == "DB_PASS" || envKey == "AWS_SECRET_ACCESS_KEY" {
// Password should be masked
if strings.Contains(logOutput, envValue) {
t.Errorf("Full password should not appear in logs: %s", logOutput)
}
// Should see at least part of the masked value
if !strings.Contains(logOutput, "...") {
t.Errorf("Expected masked password in logs, got: %s", logOutput)
}
} else if !strings.Contains(logOutput, envValue) {
t.Errorf("Expected %s in logs, but it was not found. Log output: %s", envValue, logOutput)
}
}
}
internal/serviceconfig/logger/config.go
+3 -1
View File
@@ -7,7 +7,7 @@ import (
type LogConfig struct {
Logger *slog.Logger
LogLevel string `env:"LOG_LEVEL"`
LogLevel string `env:"LOG_LEVEL" envDefault:"INFO"`
}
type ConfigProvider interface {
@@ -43,4 +43,6 @@ func (b *LogConfig) SetDefaultLogger() {
}))
slog.SetDefault(b.Logger)
slog.Info("Logger initialized", "level", b.LogLevel)
}
internal/serviceconfig/logger/readme.md
+8
View File
@@ -0,0 +1,8 @@
# Logger info
To set the log level of the default logger that will be used
throughout all of the services set the
```bash
LOG_LEVEL= DEBUG | INFO | WARN | ERROR
```
and if not specified, `INFO` is the default.
internal/serviceconfig/observability/prometheus/prometheus.go
+183
View File
@@ -0,0 +1,183 @@
package prometheus
import (
"context"
"fmt"
"net/http"
"sync"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
// MetricType represents the type of Prometheus metric
type MetricType string
const (
Counter MetricType = "counter"
Gauge MetricType = "gauge"
Histogram MetricType = "histogram"
Summary MetricType = "summary"
)
// MetricDefinition defines the structure for creating a new metric
type MetricDefinition struct {
Type MetricType
Name string
Help string
Labels []string
Buckets []float64
// Objectives is needed for Prometheus Summary metrics to define quantile calculations
Objectives map[float64]float64
}
// Metrics represents the wrapper around Prometheus metrics
type Metrics struct {
metrics map[string]interface{}
mu sync.RWMutex
server *http.Server
}
// MetricsConfig represents the configuration for the metrics wrapper
type MetricsConfig struct {
Definitions []MetricDefinition
Namespace string
Subsystem string
Port int
ServeHttp bool
}
// New creates a new Metrics instance
func New(ctx context.Context, config MetricsConfig) (*Metrics, error) {
m := &Metrics{
metrics: make(map[string]interface{}),
}
for _, def := range config.Definitions {
var metric interface{}
var err error
switch def.Type {
case Counter:
metric = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: config.Namespace,
Subsystem: config.Subsystem,
Name: def.Name,
Help: def.Help,
},
def.Labels,
)
case Gauge:
metric = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: config.Namespace,
Subsystem: config.Subsystem,
Name: def.Name,
Help: def.Help,
},
def.Labels,
)
case Histogram:
opts := prometheus.HistogramOpts{
Namespace: config.Namespace,
Subsystem: config.Subsystem,
Name: def.Name,
Help: def.Help,
}
if len(def.Buckets) > 0 {
opts.Buckets = def.Buckets
}
metric = prometheus.NewHistogramVec(opts, def.Labels)
case Summary:
opts := prometheus.SummaryOpts{
Namespace: config.Namespace,
Subsystem: config.Subsystem,
Name: def.Name,
Help: def.Help,
}
if def.Objectives != nil {
opts.Objectives = def.Objectives
}
metric = prometheus.NewSummaryVec(opts, def.Labels)
default:
return nil, fmt.Errorf("unsupported metric type: %s", def.Type)
}
collector, ok := metric.(prometheus.Collector)
if !ok {
return nil, fmt.Errorf("metric %s is not a valid prometheus collector", def.Name)
}
if err = prometheus.Register(collector); err != nil {
return nil, fmt.Errorf("failed to register metric %s: %w", def.Name, err)
}
m.metrics[def.Name] = metric
}
if config.ServeHttp {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
m.server = &http.Server{
Addr: fmt.Sprintf(":%d", config.Port),
Handler: mux,
}
go func() {
if err := m.server.ListenAndServe(); err != http.ErrServerClosed {
fmt.Printf("Metrics server error: %v\n", err)
}
}()
// Listen for context cancellation
go func() {
<-ctx.Done()
if err := m.server.Shutdown(context.Background()); err != nil {
fmt.Printf("Error shutting down metrics server: %v\n", err)
}
}()
}
return m, nil
}
// Shutdown gracefully shuts down the metrics server
func (m *Metrics) Shutdown(ctx context.Context) error {
if m.server != nil {
return m.server.Shutdown(ctx)
}
return nil
}
// RecordMetric records a value for a given metric
func (m *Metrics) RecordMetric(name string, value float64, labels map[string]string) error {
m.mu.RLock()
metric, exists := m.metrics[name]
m.mu.RUnlock()
if !exists {
return fmt.Errorf("metric %s not found", name)
}
labelValues := make([]string, 0, len(labels))
for _, v := range labels {
labelValues = append(labelValues, v)
}
switch m := metric.(type) {
case *prometheus.CounterVec:
m.WithLabelValues(labelValues...).Add(value)
case *prometheus.GaugeVec:
m.WithLabelValues(labelValues...).Set(value)
case *prometheus.HistogramVec:
m.WithLabelValues(labelValues...).Observe(value)
case *prometheus.SummaryVec:
m.WithLabelValues(labelValues...).Observe(value)
default:
return fmt.Errorf("unknown metric type for %s", name)
}
return nil
}
internal/serviceconfig/observability/prometheus/prometheus.md
+40
View File
@@ -0,0 +1,40 @@
# Prometheus docs
All instructions assume you are already in the `devbox shell`
To run various services and inspect prometheus metrics first run the
`task compose:up:test` to start the db and prometheus services.
Then you can run your service that exposed the metrics.
## Testing the API service
Start the API server on the host like
`DB_PORT=5430 go run ./cmd/queryService`
Once started you will be able to see the raw metrics at
http://localhost:8080/metrics
The prometheus console will be running at
http://localhost:9090/
The docs to the api server will be available at
http://localhost:8080/swagger/index.html
For example to run the query endpoint (to see query related metrics generated) go to
http://localhost:8080/query
## Sample prometheus queries for the console
Duration of the /query api call (from Echo metrics)
`echo_request_duration_seconds_sum{code="200", host="localhost:8080", instance="host.docker.internal:8080", job="api", method="GET", url="/query"}`
Memory allocation rate (from Go metrics)
`rate(go_memstats_alloc_bytes_total[5m])`
Time since last GC
`time() - go_memstats_last_gc_time_seconds`
Gc overhead
`rate(go_gc_duration_seconds_sum[5m])`
More examples TBD.
internal/serviceconfig/observability/prometheus/prometheus_test.go
+208
View File
@@ -0,0 +1,208 @@
package prometheus
import (
"context"
"fmt"
"io"
"net/http"
"strings"
"testing"
"time"
)
func TestMetricsIntegration(t *testing.T) {
// Create a context with cancellation for the test
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Step 1: Define our test metrics configuration
config := MetricsConfig{
Namespace: "test",
Subsystem: "metrics",
Port: 9090, // Using 9090 for tests
ServeHttp: true, // Enable the built-in HTTP server
Definitions: []MetricDefinition{
{
// Counter example: counting total events
Type: Counter,
Name: "events_total",
Help: "Total number of events processed",
Labels: []string{"type", "status"},
},
{
// Gauge example: tracking current value
Type: Gauge,
Name: "queue_size",
Help: "Current number of items in queue",
Labels: []string{"queue_name"},
},
{
// Histogram example: measuring distribution of values
Type: Histogram,
Name: "response_time_seconds",
Help: "Response time distribution",
Labels: []string{"endpoint"},
Buckets: []float64{0.1, 0.5, 1, 2, 5}, // Define bucket boundaries
},
{
// Summary example: calculating quantiles
Type: Summary,
Name: "request_size_bytes",
Help: "Request size distribution",
Labels: []string{"method"},
Objectives: map[float64]float64{
0.5: 0.05, // 50th percentile with 5% error
0.9: 0.01, // 90th percentile with 1% error
0.99: 0.001, // 99th percentile with 0.1% error
},
},
},
}
// Step 2: Create new metrics instance with context
metrics, err := New(ctx, config)
if err != nil {
t.Fatalf("Failed to create metrics: %v", err)
}
// Ensure cleanup after test
defer func() {
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
defer shutdownCancel()
if err := metrics.Shutdown(shutdownCtx); err != nil {
t.Errorf("Failed to shutdown metrics server: %v", err)
}
}()
// Step 3: Record some sample metrics
// Record a counter metric
err = metrics.RecordMetric("events_total", 42, map[string]string{
"type": "user_login",
"status": "success",
})
if err != nil {
t.Fatalf("Failed to record counter metric: %v", err)
}
// Record a gauge metric
err = metrics.RecordMetric("queue_size", 15, map[string]string{
"queue_name": "default",
})
if err != nil {
t.Fatalf("Failed to record gauge metric: %v", err)
}
// Record a histogram metric
err = metrics.RecordMetric("response_time_seconds", 0.7, map[string]string{
"endpoint": "/api/users",
})
if err != nil {
t.Fatalf("Failed to record histogram metric: %v", err)
}
// Record a summary metric
err = metrics.RecordMetric("request_size_bytes", 1024, map[string]string{
"method": "POST",
})
if err != nil {
t.Fatalf("Failed to record summary metric: %v", err)
}
// Step 4: Wait for HTTP server to start
// Give the server a moment to start up
time.Sleep(time.Second)
// Step 5: Fetch metrics from the HTTP endpoint
resp, err := http.Get("http://localhost:9090/metrics")
if err != nil {
t.Fatalf("Failed to fetch metrics: %v", err)
}
defer resp.Body.Close()
// Read the response body
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("Failed to read response body: %v", err)
}
metricsOutput := string(body)
// if needed for debugging.
//log.Infof("Metrics output: %s", metricsOutput)
// Step 6: Verify that our custom metrics and go metrics are present
expectedMetrics := []string{
// Our custom metrics (with namespace and subsystem)
"test_metrics_events_total",
"test_metrics_queue_size",
"test_metrics_response_time_seconds",
"test_metrics_request_size_bytes",
// Standard Go metrics that should be present
"go_memstats_alloc_bytes_total",
}
// Check for each expected metric
for _, metric := range expectedMetrics {
if !strings.Contains(metricsOutput, metric) {
t.Errorf("Expected metric %s not found in output", metric)
// Print a snippet of the output around where we expect to find the metric
fmt.Printf("Searching for: %s\nOutput snippet: %s\n",
metric,
getOutputSnippet(metricsOutput, metric))
}
}
// Test cancellation
t.Run("Context cancellation", func(t *testing.T) {
// Create a new metrics instance with a separate context
cancelCtx, cancelFunc := context.WithCancel(context.Background())
_, err := New(cancelCtx, MetricsConfig{
Namespace: "test_cancel",
Port: 9091,
ServeHttp: true,
})
if err != nil {
t.Fatalf("Failed to create metrics for cancellation test: %v", err)
}
// Wait for server to start
time.Sleep(time.Second)
// Cancel the context
cancelFunc()
// Try to connect to the server - it should fail after a short delay
time.Sleep(time.Second)
_, err = http.Get("http://localhost:9091/metrics")
if err == nil {
t.Error("Expected server to be shutdown after context cancellation")
}
})
}
// Helper function to get a snippet of the output around where we expect to find the metric
func getOutputSnippet(output string, searchTerm string) string {
index := strings.Index(output, searchTerm)
if index == -1 {
return "Metric not found in output"
}
// Get up to 200 characters before and after the search term
start := max(0, index-200)
end := min(len(output), index+200)
return output[start:end]
}
func max(a, b int) int {
if a > b {
return a
}
return b
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
internal/serviceconfig/observability/prometheus/readme.md
+264
View File
@@ -0,0 +1,264 @@
# Prometheus (Doczy)
Prometheus is a simplified wrapper around the Prometheus client library for Go, making it easier to define and use Prometheus metrics in your services. It provides a straightforward way to create and record various types of Prometheus metrics with minimal boilerplate code.
## Features
- Simple configuration-based metric definition
- Support for all major Prometheus metric types:
- Counter
- Gauge
- Histogram
- Summary
- Automatic HTTP metrics endpoint setup
- Thread-safe metric recording
- Label support for all metric types
- Context-based graceful shutdown
- Clean server lifecycle management
## Example cmd (metricsExample_test)
See `cmd/metricsExample_test` for a sample usage in a working main.go.
The Readme.md in that directory has more details on how to use the harness.
## Running tests with prometheus server
See [prometheus.md](./prometheus.md) for prometheus related instructions and testing instructions.
## Quick Start (with your own http server)
This example (and the `metricsExample_test`) shows how to use the harness to create a simple http server that exposes metrics.
If you are using this with an API server you can indicate in your config that you already have a http server and it will not start a http server for you.
(e.g. Use the one that we automatically setup as Echo middleware)
For a more complete and up-to-date example always refer to the `metricsExample_test` sample which you can run.
```go
package main
import (
"context"
"log"
"os"
"os/signal"
"syscall"
"time"
"queryorchestration/internal/serviceconfig/prometheus"
)
func main() {
// Create a context that we can cancel
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
config := prometrics.Config{
Namespace: "myapp",
Subsystem: "subsystem",
ServeHttp: true,
Port: 8080,
Definitions: []prometheus.MetricDefinition{
{
Type: prometheus.Counter,
Name: "requests_total",
Help: "Total number of requests processed",
Labels: []string{"endpoint", "status"},
},
},
}
metrics, err := prometheus.New(ctx, config)
if err != nil {
panic(err)
}
// Setup graceful shutdown
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sigChan
// Cancel context
cancel()
// Give the server up to 5 seconds to shutdown gracefully
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
defer shutdownCancel()
if err := metrics.Shutdown(shutdownCtx); err != nil {
log.Printf("Error shutting down metrics server: %v", err)
}
}()
// Record a metric
metrics.RecordMetric("requests_total", 1, map[string]string{
"endpoint": "/api/v1",
"status": "success",
})
// Block until signal is received
<-ctx.Done()
}
```
## Configuration
### MetricsConfig Structure
```go
type MetricsConfig struct {
Definitions []MetricDefinition // List of metrics to create
Namespace string // Namespace for all metrics
Subsystem string // Subsystem for all metrics
ServeHttp bool // Whether to serve metrics endpoint
Port int // Port for metrics endpoint
}
```
### MetricDefinition Structure
```go
type MetricDefinition struct {
Type MetricType // Counter, Gauge, Histogram, or Summary
Name string // Metric name
Help string // Help text for the metric
Labels []string // Label names for the metric
Buckets []float64 // Buckets for Histogram type
Objectives map[float64]float64 // Objectives for Summary type
}
```
## Metric Types
### Counter
Used for cumulative metrics that only increase in value.
```go
MetricDefinition{
Type: prometrics.Counter,
Name: "requests_total",
Help: "Total requests processed",
Labels: []string{"endpoint"},
}
```
### Gauge
Used for metrics that can increase and decrease.
```go
MetricDefinition{
Type: prometrics.Gauge,
Name: "active_connections",
Help: "Current number of active connections",
Labels: []string{"server"},
}
```
### Histogram
Used for measurements like request duration or response size.
```go
MetricDefinition{
Type: prometrics.Histogram,
Name: "request_duration_seconds",
Help: "Request duration in seconds",
Labels: []string{"endpoint"},
Buckets: []float64{0.1, 0.5, 1, 2, 5},
}
```
### Summary
Similar to histogram but calculates quantiles over time.
```go
MetricDefinition{
Type: prometrics.Summary,
Name: "request_latency",
Help: "Request latency",
Labels: []string{"endpoint"},
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
}
```
## Recording Metrics
Use the `RecordMetric` method to record values for any metric type:
```go
err := metrics.RecordMetric("metric_name", value, map[string]string{
"label1": "value1",
"label2": "value2",
})
```
The method is thread-safe and will return an error if:
- The metric doesn't exist
- The label values don't match the metric's label names
- The metric type is invalid
## HTTP Endpoint
When `ServeHttp` is set to `true` in the configuration, the package automatically serves metrics at `/metrics` on the specified port. This endpoint can be scraped by Prometheus.
## Lifecycle Management
The library provides two ways to manage the lifecycle of the metrics server:
1. Context-based shutdown:
```go
ctx, cancel := context.WithCancel(context.Background())
metrics, err := prometheus.New(ctx, config)
// Server will shut down when context is cancelled
cancel()
```
2. Manual shutdown:
```go
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
defer shutdownCancel()
err := metrics.Shutdown(shutdownCtx)
```
The metrics server will also automatically shut down when the main process exits, but it's recommended to use one of the above methods for graceful shutdown.
## Thread Safety
All operations in the package are thread-safe, allowing for concurrent metric recording from multiple goroutines.
## Error Handling
The package returns errors in the following cases:
- Invalid metric type specified in configuration
- Failed metric registration (e.g., duplicate metrics)
- Recording to non-existent metric
- Invalid label combinations
- Server shutdown failures
## Best Practices
1. Always provide a context when creating new metrics:
```go
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
metrics, err := prometheus.New(ctx, config)
```
2. Implement graceful shutdown in your application:
```go
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
defer shutdownCancel()
if err := metrics.Shutdown(shutdownCtx); err != nil {
log.Printf("Error during metrics shutdown: %v", err)
}
```
3. Handle server errors appropriately:
```go
if err := metrics.Shutdown(ctx); err != nil {
// Log the error but don't panic - the application can continue
log.Printf("Metrics server shutdown error: %v", err)
}
```
4. Use appropriate timeouts for shutdown:
```go
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
metrics.Shutdown(ctx)
```