Observability #

// 1. Counters — only increase, never decrease
// Fits: request counts, error counts, event counts
httpRequestsTotal := prometheus.NewCounterVec(
    prometheus.CounterOpts{
        Name: "http_requests_total",
        Help: "Total HTTP requests",
    },
    []string{"method", "endpoint", "status_code"}, // labels for dimensions
)

// Usage:
httpRequestsTotal.WithLabelValues("GET", "/api/products", "200").Inc()

// 2. Gauges — can go up and down
// Fits: active connections, queue sizes, memory usage
activeConnections := prometheus.NewGauge(prometheus.GaugeOpts{
    Name: "active_connections",
    Help: "Number of active connections",
})

queueSize := prometheus.NewGaugeVec(
    prometheus.GaugeOpts{
        Name: "queue_size",
        Help: "Current number of items in queue",
    },
    []string{"queue_name"},
)

activeConnections.Inc()    // up
activeConnections.Dec()    // down
activeConnections.Set(42)  // set directly

// 3. Histograms — value distributions (for latency)
// Fits: latencies, request/response sizes
requestDurationSeconds := prometheus.NewHistogramVec(
    prometheus.HistogramOpts{
        Name:    "http_request_duration_seconds",
        Help:    "HTTP request duration",
        Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5},
        // buckets = time boundaries in seconds (5ms, 10ms, 25ms, ...)
    },
    []string{"method", "endpoint"},
)

// Usage — distributions calculated automatically
timer := prometheus.NewTimer(
    requestDurationSeconds.WithLabelValues("GET", "/api/products"),
)
result := fetchProducts()
timer.ObserveDuration()

// 4. Summaries — like Histograms but quantiles calculated client-side
// Rarely used because they can't be aggregated across instances
requestProcessingSeconds := prometheus.NewSummary(prometheus.SummaryOpts{
    Name: "request_processing_seconds",
    Help: "Time spent processing request",
})

// Go: net/http middleware equivalent
// Metrics
var (
    requestCount = prometheus.NewCounterVec(
        prometheus.CounterOpts{
            Name: "flask_request_count",
            Help: "Flask Request Count",
        },
        []string{"method", "endpoint", "status"},
    )

    requestLatency = prometheus.NewHistogramVec(
        prometheus.HistogramOpts{
            Name:    "flask_request_latency_seconds",
            Help:    "Flask Request Latency",
            Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0},
        },
        []string{"method", "endpoint"},
    )
)

// statusRecorder captures the response status code
type statusRecorder struct {
    http.ResponseWriter
    status int
}

// Middleware — runs before and after every request
func metricsMiddleware(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        start := time.Now()

        rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
        next.ServeHTTP(rec, r)

        requestCount.WithLabelValues(
            r.Method, r.URL.Path, strconv.Itoa(rec.status),
        ).Inc()
        requestLatency.WithLabelValues(r.Method, r.URL.Path).
            Observe(time.Since(start).Seconds())
    })
}

// The endpoint for Prometheus scraping
func metricsHandler(w http.ResponseWriter, r *http.Request) {
    w.Header().Set("Content-Type", "text/plain; charset=utf-8")
    promhttp.Handler().ServeHTTP(w, r)
}

// Go: standard library log/slog equivalent
// requestID carries the request ID through the request context
type ctxKey string

const requestIDKey ctxKey = "request_id"

// structuredFormatter emits one JSON object per log line
func structuredFormatter(r slog.Record) ([]byte, error) {
    attrs := make(map[string]any)
    attrs["timestamp"] = r.Time.Format("2006-01-02T15:04:05.000")
    attrs["level"] = r.Level.String()
    attrs["logger"] = "myapp"
    attrs["message"] = r.Message
    attrs["request_id"] = ""
    attrs["service"] = "myapp"
    attrs["environment"] = os.Getenv("APP_ENV")

    r.Attrs(func(a slog.Attr) bool {
        attrs[a.Key] = a.Value.Any()
        return true
    })

    return json.Marshal(attrs)
}

// Middleware for injecting request IDs
func injectRequestID(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        requestID := r.Header.Get("X-Request-ID")
        if requestID == "" {
            requestID = uuid.New().String()
        }
        // Also set it in the response header for client tracing
        w.Header().Set("X-Request-ID", requestID)
        next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), requestIDKey, requestID)))
    })
}

// logRequest logs a structured line after every request
func logRequest(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        start := time.Now()
        next.ServeHTTP(w, r)

        logger.Info("HTTP request completed",
            "method", r.Method,
            "path", r.URL.Path,
            "duration_ms", float64(time.Since(start).Microseconds())/1000,
            "request_id", r.Context().Value(requestIDKey),
        )
    })
}

// Go: OpenTelemetry Go SDK equivalent
func setupTracing(serviceName string, otlpEndpoint string) {
    // serviceName and otlpEndpoint identify this service and the collector
    ctx := context.Background()

    exporter, err := otlptracegrpc.New(ctx,
        otlptracegrpc.WithEndpoint(otlpEndpoint),
        otlptracegrpc.WithInsecure(),
    )
    if err != nil {
        log.Fatal(err)
    }

    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exporter),
        sdktrace.WithResource(resource.NewWithAttributes(
            semconv.SchemaURL,
            semconv.ServiceName(serviceName),
            attribute.String("service.version", os.Getenv("APP_VERSION")),
            attribute.String("deployment.environment", os.Getenv("APP_ENV")),
        )),
    )
    otel.SetTracerProvider(tp)
}

// Tracer for manual instrumentation
var tracer = otel.Tracer("myapp")

// Manual span examples
func processOrder(ctx context.Context, orderData map[string]any) (map[string]any, error) {
    ctx, span := tracer.Start(ctx, "process_order")
    defer span.End()
    // Add useful attributes for debugging
    span.SetAttributes(
        attribute.String("order.id", orderData["id"].(string)),
        attribute.Float64("order.total", orderData["total"].(float64)),
        attribute.Int("order.item_count", len(orderData["items"].([]any))),
    )

    ctx, inventorySpan := tracer.Start(ctx, "validate_inventory")
    inventoryOK := checkInventory(orderData["items"])
    if !inventoryOK {
        inventorySpan.RecordError(errors.New("Insufficient inventory"))
        inventorySpan.SetStatus(codes.Error, "Insufficient inventory")
        inventorySpan.End()
        return nil, errors.New("insufficient inventory")
    }
    inventorySpan.End()

    _, paymentSpan := tracer.Start(ctx, "process_payment")
    paymentSpan.SetAttributes(attribute.Float64("payment.amount", orderData["total"].(float64)))
    result := chargePayment(orderData)
    paymentSpan.SetAttributes(attribute.String("payment.transaction_id", result["transaction_id"].(string)))
    paymentSpan.End()

    return finalizeOrder(orderData, result)
}

// Go: calculating and tracking SLI/SLO
// SLI 1: Availability (percentage of successful requests)
httpRequestsTotal := prometheus.NewCounterVec(
    prometheus.CounterOpts{
        Name: "http_requests_total",
        Help: "Total HTTP requests",
    },
    []string{"status_code"},
)

// Availability = successful requests / total requests
// Prometheus query:
// sum(rate(http_requests_total{status_code!~"5.."}[30d]))
// /
// sum(rate(http_requests_total[30d]))
func calculateAvailabilitySLI() float64 {
    // Implemented via the Prometheus API
    return 0.0
}

// SLI 2: Latency (percentage of requests under thresholds)
requestDuration := prometheus.NewHistogram(prometheus.HistogramOpts{
    Name:    "http_request_duration_seconds",
    Help:    "Request duration",
    Buckets: []float64{0.1, 0.2, 0.5, 1.0, 2.0, 5.0},
})

// Latency SLI query:
// sum(rate(http_request_duration_seconds_bucket{le="0.2"}[30d]))
// /
// sum(rate(http_request_duration_seconds_count[30d]))
// → The percentage of requests completing within 200ms

// Alerts for SLO breaches and error budget burn rates

// Go: anti-patterns
// ✗ Anti-pattern 1: logs as the only observability
// No metrics, no tracing
// Debugging requires manual greps through thousands of log lines
// ✓ Solution: implement all three pillars — metrics, logs, traces

// ✗ Anti-pattern 2: unstructured logs
logger.Printf("User %d failed to login from %s after %d attempts", userID, ip, n)
// Can't be queried, aggregated, or filtered
// ✓ Solution: structured JSON logs with consistent fields

// ✗ Anti-pattern 3: alerting on non-actionable metrics
// alert: CPU > 80%
// What should be done? Who should be contacted?
// ✓ Solution: alert on symptoms (latency, error rates) not causes

// ✗ Anti-pattern 4: alert fatigue — too many alerts
// Teams get 200 alerts a day, all ignored
// ✓ Solution: audit alerts regularly, remove ones never actionable

// ✗ Anti-pattern 5: no runbooks for alerts
// Alerts go off, on-call doesn't know what to do
// ✓ Solution: every alert has a clear runbook link

// ✗ Anti-pattern 6: trace IDs not propagated
// Traces start at API gateways but aren't forwarded to other services
// Traces fragment, can't be viewed end-to-end
// ✓ Solution: OpenTelemetry auto-instrumentation propagating trace contexts

// ✗ Anti-pattern 7: dashboards with too many graphs
// 50+ panels on one dashboard, hard to focus during incidents
// ✓ Solution: concise dashboards for on-call, detailed dashboards for debugging