Observability #

from prometheus_client import Counter, Gauge, Histogram, Summary, start_http_server
import time

# 1. Counter — hanya naik, tidak bisa turun
# Cocok untuk: jumlah request, jumlah error, jumlah event
http_requests_total = Counter(
    'http_requests_total',
    'Total HTTP requests',
    ['method', 'endpoint', 'status_code']  # labels untuk dimensi
)

# Penggunaan:
http_requests_total.labels(
    method='GET',
    endpoint='/api/products',
    status_code='200'
).inc()

# 2. Gauge — bisa naik dan turun
# Cocok untuk: koneksi aktif, ukuran queue, penggunaan memory
active_connections = Gauge(
    'active_connections',
    'Number of active connections'
)
queue_size = Gauge(
    'queue_size',
    'Current number of items in queue',
    ['queue_name']
)

active_connections.inc()    # naik
active_connections.dec()    # turun
active_connections.set(42)  # set langsung

# 3. Histogram — distribusi nilai (untuk latency)
# Cocok untuk: latency, ukuran request/response
request_duration_seconds = Histogram(
    'http_request_duration_seconds',
    'HTTP request duration',
    ['method', 'endpoint'],
    buckets=[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]
    # buckets = batas waktu dalam detik (5ms, 10ms, 25ms, ...)
)

# Penggunaan — otomatis hitung distribusi
with request_duration_seconds.labels(
    method='GET', endpoint='/api/products'
).time():
    result = fetch_products()

# 4. Summary — seperti Histogram tapi hitung quantile di client
# Jarang digunakan karena tidak bisa di-aggregate antar instance
request_processing_seconds = Summary(
    'request_processing_seconds',
    'Time spent processing request'
)

from flask import Flask, request, g
from prometheus_client import Counter, Histogram, generate_latest
import time

app = Flask(__name__)

# Metrics
REQUEST_COUNT = Counter(
    'flask_request_count',
    'Flask Request Count',
    ['method', 'endpoint', 'status']
)

REQUEST_LATENCY = Histogram(
    'flask_request_latency_seconds',
    'Flask Request Latency',
    ['method', 'endpoint'],
    buckets=[0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
)

@app.before_request
def start_timer():
    g.start_time = time.time()

@app.after_request
def record_request_metrics(response):
    latency = time.time() - g.start_time

    REQUEST_COUNT.labels(
        method=request.method,
        endpoint=request.endpoint or 'unknown',
        status=response.status_code
    ).inc()

    REQUEST_LATENCY.labels(
        method=request.method,
        endpoint=request.endpoint or 'unknown'
    ).observe(latency)

    return response

# Endpoint untuk Prometheus scrape
@app.route('/metrics')
def metrics():
    return generate_latest(), 200, {'Content-Type': 'text/plain; charset=utf-8'}

# prometheus.yml — konfigurasi scraping

global:
  scrape_interval: 15s      # scrape setiap 15 detik
  evaluation_interval: 15s  # evaluasi alert setiap 15 detik

scrape_configs:
  - job_name: 'myapp'
    static_configs:
      - targets: ['app:8000']  # host:port endpoint /metrics
    metrics_path: '/metrics'

  - job_name: 'node-exporter'  # sistem metrics (CPU, memory, disk)
    static_configs:
      - targets: ['node-exporter:9100']

import logging
import json
import time
import uuid
from contextvars import ContextVar

# Context variable untuk request ID (thread-safe)
request_id_var: ContextVar[str] = ContextVar('request_id', default='')

class StructuredFormatter(logging.Formatter):
    """Format log sebagai JSON untuk kemudahan parsing."""

    def format(self, record: logging.LogRecord) -> str:
        log_data = {
            'timestamp': self.formatTime(record, '%Y-%m-%dT%H:%M:%S.%f'),
            'level': record.levelname,
            'logger': record.name,
            'message': record.getMessage(),
            'request_id': request_id_var.get(''),
            'service': 'myapp',
            'environment': os.environ.get('APP_ENV', 'development'),
        }

        # Tambahkan extra fields jika ada
        if hasattr(record, 'extra'):
            log_data.update(record.extra)

        # Tambahkan exception info jika ada
        if record.exc_info:
            log_data['exception'] = self.formatException(record.exc_info)

        return json.dumps(log_data)

# Setup logging
def setup_logging():
    handler = logging.StreamHandler()
    handler.setFormatter(StructuredFormatter())

    root_logger = logging.getLogger()
    root_logger.addHandler(handler)
    root_logger.setLevel(logging.INFO)

# Logger yang digunakan di seluruh aplikasi
logger = logging.getLogger(__name__)

# Middleware untuk inject request ID
from flask import Flask, request, g

app = Flask(__name__)

@app.before_request
def inject_request_id():
    request_id = request.headers.get('X-Request-ID', str(uuid.uuid4()))
    g.request_id = request_id
    request_id_var.set(request_id)
    # Set juga di response header untuk client tracing
    g.start_time = time.time()

@app.after_request
def log_request(response):
    duration = time.time() - g.start_time
    logger.info(
        "HTTP request completed",
        extra={
            'method': request.method,
            'path': request.path,
            'status_code': response.status_code,
            'duration_ms': round(duration * 1000, 2),
            'request_id': g.request_id,
            'user_agent': request.user_agent.string[:100],
            'ip': request.remote_addr,
        }
    )
    response.headers['X-Request-ID'] = g.request_id
    return response

from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.flask import FlaskInstrumentor
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
from opentelemetry.instrumentation.redis import RedisInstrumentor
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor

def setup_tracing(service_name: str, otlp_endpoint: str = "http://jaeger:4317"):
    """Setup OpenTelemetry tracing."""

    provider = TracerProvider(
        resource=Resource.create({
            "service.name": service_name,
            "service.version": os.environ.get("APP_VERSION", "unknown"),
            "deployment.environment": os.environ.get("APP_ENV", "development"),
        })
    )

    # Kirim traces ke Jaeger via OTLP
    exporter = OTLPSpanExporter(endpoint=otlp_endpoint)
    provider.add_span_processor(BatchSpanProcessor(exporter))
    trace.set_tracer_provider(provider)

    # Auto-instrument libraries
    FlaskInstrumentor().instrument()
    SQLAlchemyInstrumentor().instrument()
    RedisInstrumentor().instrument()
    HTTPXClientInstrumentor().instrument()

# Tracer untuk manual instrumentation
tracer = trace.get_tracer(__name__)

# Contoh manual span
def process_order(order_data: dict) -> dict:
    with tracer.start_as_current_span("process_order") as span:
        # Tambahkan attribute yang berguna untuk debugging
        span.set_attribute("order.id", order_data["id"])
        span.set_attribute("order.total", order_data["total"])
        span.set_attribute("order.item_count", len(order_data["items"]))

        with tracer.start_as_current_span("validate_inventory"):
            inventory_ok = check_inventory(order_data["items"])
            if not inventory_ok:
                span.set_status(trace.StatusCode.ERROR, "Insufficient inventory")
                raise InsufficientInventoryError()

        with tracer.start_as_current_span("process_payment") as payment_span:
            payment_span.set_attribute("payment.amount", order_data["total"])
            result = charge_payment(order_data)
            payment_span.set_attribute("payment.transaction_id", result["transaction_id"])

        return finalize_order(order_data, result)

# Menghitung dan tracking SLI/SLO

from prometheus_client import Histogram, Counter

# SLI 1: Availability (persentase request yang berhasil)
http_requests_total = Counter(
    'http_requests_total',
    'Total HTTP requests',
    ['status_code']
)

def calculate_availability_sli(time_range='30d') -> float:
    """
    Availability = successful requests / total requests
    Query Prometheus:
    sum(rate(http_requests_total{status_code!~"5.."}[30d]))
    /
    sum(rate(http_requests_total[30d]))
    """
    pass  # Implementasi via Prometheus API

# SLI 2: Latency (persentase request di bawah threshold)
request_duration = Histogram(
    'http_request_duration_seconds',
    'Request duration',
    buckets=[0.1, 0.2, 0.5, 1.0, 2.0, 5.0]
)

# Query untuk latency SLI:
# sum(rate(http_request_duration_seconds_bucket{le="0.2"}[30d]))
# /
# sum(rate(http_request_duration_seconds_count[30d]))
# → Persentase request yang selesai dalam 200ms

# Alert untuk SLO breach dan error budget burn rate

# Prometheus alerting rules untuk SLO

groups:
  - name: slo_alerts
    rules:
      # Alert jika error rate melebihi threshold
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status_code=~"5.."}[5m]))
          /
          sum(rate(http_requests_total[5m])) > 0.05          
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Error rate > 5% selama 5 menit"
          description: "Error rate: {{ $value | humanizePercentage }}"

      # Alert untuk latency SLO
      - alert: HighLatency
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
          ) > 1.0          
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "P99 latency > 1 detik"
          description: "P99 latency: {{ $value }}s"

      # Alert untuk error budget burn rate yang terlalu cepat
      - alert: ErrorBudgetBurnRateFast
        expr: |
          (
            sum(rate(http_requests_total{status_code=~"5.."}[1h]))
            /
            sum(rate(http_requests_total[1h]))
          ) > 0.001 * 14.4
          # 14.4x normal burn rate = akan habis dalam 5 hari          
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Error budget burn rate terlalu cepat"

# Alertmanager routing — kirim alert ke channel yang tepat

global:
  resolve_timeout: 5m

route:
  group_by: ['alertname', 'cluster']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'slack-default'

  routes:
    # Critical alerts → PagerDuty (bangunkan on-call)
    - match:
        severity: critical
      receiver: 'pagerduty-oncall'
      continue: false

    # Warning alerts → Slack
    - match:
        severity: warning
      receiver: 'slack-alerts'

receivers:
  - name: 'pagerduty-oncall'
    pagerduty_configs:
      - service_key: ${PAGERDUTY_KEY}
        description: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'

  - name: 'slack-alerts'
    slack_configs:
      - api_url: ${SLACK_WEBHOOK}
        channel: '#alerts'
        text: |
          *Alert:* {{ .GroupLabels.alertname }}
          *Summary:* {{ range .Alerts }}{{ .Annotations.summary }}{{ end }}
          *Runbook:* {{ range .Alerts }}{{ .Annotations.runbook_url }}{{ end }}          

# ✗ Anti-pattern 1: Log sebagai satu-satunya observability
# Tidak ada metrics, tidak ada tracing
# Debugging butuh grep manual di ribuan baris log
# ✓ Solusi: implementasi ketiga pilar — metrics, logs, traces

# ✗ Anti-pattern 2: Unstructured log
logger.info(f"User {user_id} failed to login from {ip} after {n} attempts")
# Tidak bisa di-query, tidak bisa di-aggregate, tidak bisa di-filter
# ✓ Solusi: structured JSON log dengan field yang konsisten

# ✗ Anti-pattern 3: Alert pada metrik yang tidak actionable
# alert: CPU > 80%
# Apa yang harus dilakukan? Siapa yang harus dihubungi?
# ✓ Solusi: alert pada symptom (latency, error rate) bukan cause

# ✗ Anti-pattern 4: Alert fatigue — terlalu banyak alert
# Tim mendapat 200 alert per hari, semua diabaikan
# ✓ Solusi: audit alert secara berkala, hapus yang tidak pernah actionable

# ✗ Anti-pattern 5: Tidak ada runbook untuk alert
# Alert berbunyi, on-call tidak tahu harus apa
# ✓ Solusi: setiap alert punya link runbook yang jelas

# ✗ Anti-pattern 6: Trace ID tidak di-propagate
# Trace mulai di API gateway tapi tidak diteruskan ke service lain
# Trace terfragmentasi, tidak bisa dilihat end-to-end
# ✓ Solusi: OpenTelemetry auto-instrumentation yang propagate trace context

# ✗ Anti-pattern 7: Dashboard yang terlalu banyak grafik
# 50+ panel di satu dashboard, sulit fokus saat insiden
# ✓ Solusi: dashboard ringkas untuk on-call, detail dashboard untuk debugging