Retry Strategy #

// ANTI-PATTERN: retry tanpa strategi — berbahaya
func callServiceDangerous() error {
    for {                          // ← tidak ada batas
        err := callService()
        if err == nil { return nil }
        // tidak ada delay — DDoS ke diri sendiri
        // tidak ada klasifikasi error — retry validation error yang tidak akan berubah
    }
}

// BENAR: retry dengan konfigurasi eksplisit
func callServiceSafe(ctx context.Context) error {
    return RetryWithBackoff(ctx, RetryConfig{
        MaxAttempts:   5,
        BaseDelay:     100 * time.Millisecond,
        MaxDelay:      30 * time.Second,
        Jitter:        true,
        IsRetryable:   isTransientError,
    }, func() error {
        return callService()
    })
}
// Error 4xx → fail fast tanpa retry
// Error 5xx/timeout → retry dengan increasing delay
// Setelah max attempts → return final error

// ANTI-PATTERN: immediate retry untuk network call
for attempt := 0; attempt < maxAttempts; attempt++ {
    err := callExternalService()
    if err == nil { return nil }
    // ← tidak ada delay
    // → saat downstream down 100ms, semua retry langsung flood bersamaan
}

// BENAR: immediate retry hanya untuk operasi in-memory yang benar-benar microsecond
for attempt := 0; attempt < 3; attempt++ {
    if acquired := mutex.TryLock(); acquired {
        return nil
    }
    // mutex contention sembuh dalam nanosecond, immediate retry aman di sini
    runtime.Gosched() // yield CPU, coba lagi segera
}

// Fixed delay — sederhana, cocok untuk sistem sederhana dengan satu instance
func retryFixed(ctx context.Context, maxAttempts int, delay time.Duration,
    fn func() error) error {

    var lastErr error
    for attempt := 1; attempt <= maxAttempts; attempt++ {
        lastErr = fn()
        if lastErr == nil {
            return nil
        }
        if attempt < maxAttempts {
            select {
            case <-time.After(delay):
            case <-ctx.Done():
                return ctx.Err()
            }
        }
    }
    return fmt.Errorf("all %d attempts failed: %w", maxAttempts, lastErr)
}

// ANTI-PATTERN: ratusan instance retry dengan delay persis sama
// → thundering herd: semua 200 instance retry setelah tepat 1 detik
// → downstream menerima spike 200 request sekaligus, bukan spread merata

// Delay: 100ms → 200ms → 400ms → 800ms → 1600ms (capped)
func calculateBackoff(attempt int, base, max time.Duration) time.Duration {
    delay := base * time.Duration(1<<uint(attempt-1)) // base × 2^(attempt-1)
    if delay > max {
        delay = max
    }
    return delay
}

// Implementasi lengkap: exponential backoff + full jitter
func RetryWithBackoff(ctx context.Context, cfg RetryConfig, fn func() error) error {
    var lastErr error

    for attempt := 1; attempt <= cfg.MaxAttempts; attempt++ {
        lastErr = fn()

        if lastErr == nil {
            if attempt > 1 {
                log.Infof("retry succeeded on attempt %d", attempt)
            }
            return nil
        }

        // Fail fast untuk error yang tidak layak di-retry
        if !cfg.IsRetryable(lastErr) {
            return fmt.Errorf("non-retryable error: %w", lastErr)
        }

        if attempt == cfg.MaxAttempts {
            break // jangan tunggu setelah attempt terakhir
        }

        // Hitung base delay: min(base × 2^(attempt-1), maxDelay)
        baseDelay := cfg.BaseDelay * time.Duration(1<<uint(attempt-1))
        if baseDelay > cfg.MaxDelay {
            baseDelay = cfg.MaxDelay
        }

        // Full jitter: random antara 0 dan baseDelay
        // Lebih efektif daripada ±25% dalam menghindari thundering herd
        jitter := time.Duration(rand.Int63n(int64(baseDelay)))

        log.Warnf("attempt %d/%d failed: %v — retrying in %v",
            attempt, cfg.MaxAttempts, lastErr, jitter)

        select {
        case <-time.After(jitter):
        case <-ctx.Done():
            return ctx.Err()
        }
    }

    return fmt.Errorf("exhausted %d attempts: %w", cfg.MaxAttempts, lastErr)
}

type RetryConfig struct {
    MaxAttempts int
    BaseDelay   time.Duration
    MaxDelay    time.Duration
    IsRetryable func(error) bool
}

// Retry sampai deadline — bukan sampai max attempt
func RetryUntilDeadline(ctx context.Context, timeout time.Duration,
    fn func() error) error {

    deadline := time.Now().Add(timeout)
    ctx, cancel := context.WithDeadline(ctx, deadline)
    defer cancel()

    attempt := 0
    baseDelay := 50 * time.Millisecond

    for {
        attempt++
        err := fn()
        if err == nil {
            return nil
        }

        remaining := time.Until(deadline)
        if remaining <= 0 {
            return fmt.Errorf("deadline exceeded after %d attempts: %w", attempt, err)
        }

        nextDelay := baseDelay * time.Duration(1<<uint(attempt-1))
        // Jangan retry jika delay berikutnya lebih dari setengah remaining time
        if nextDelay > remaining/2 {
            return fmt.Errorf("insufficient time budget for retry: %w", err)
        }

        jitter := time.Duration(rand.Int63n(int64(nextDelay)))
        select {
        case <-time.After(jitter):
        case <-ctx.Done():
            return ctx.Err()
        }
    }
}

// Fungsi klasifikasi error yang eksplisit
func isRetryableError(err error) bool {
    if err == nil {
        return false
    }

    // Context cancelled atau deadline exceeded — jangan retry
    if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
        return false
    }

    var httpErr *HTTPError
    if errors.As(err, &httpErr) {
        switch httpErr.StatusCode {
        // ✓ BOLEH retry — error transient
        case 429: return true // Too Many Requests — tunggu dan coba lagi
        case 500: return true // Internal Server Error — mungkin transient
        case 502: return true // Bad Gateway — upstream sedang restart
        case 503: return true // Service Unavailable — downstream overload
        case 504: return true // Gateway Timeout

        // ✗ JANGAN retry — error deterministic
        case 400: return false // Bad Request — request salah, tidak akan berubah
        case 401: return false // Unauthorized — perlu refresh token dulu
        case 403: return false // Forbidden — tidak punya akses
        case 404: return false // Not Found — resource tidak ada
        case 409: return false // Conflict — butuh logic khusus, bukan sekedar retry
        case 422: return false // Unprocessable Entity — data tidak valid
        }
    }

    // Network errors transient — biasanya layak di-retry
    var netErr *net.OpError
    if errors.As(err, &netErr) {
        return true
    }

    return false // default: jangan retry jika tidak yakin
}

// ANTI-PATTERN: retry operasi yang tidak idempotent
func processPayment(userID string, amount int64) error {
    return RetryWithBackoff(ctx, defaultConfig, func() error {
        // Jika request berhasil tapi response timeout sebelum sampai,
        // retry akan memproses PAYMENT KEDUA — double charge!
        return paymentGateway.Charge(userID, amount)
    })
}

// BENAR: idempotency key memastikan retry selalu aman
func processPayment(idempotencyKey, userID string, amount int64) error {
    return RetryWithBackoff(ctx, defaultConfig, func() error {
        return paymentGateway.Charge(ChargeRequest{
            IdempotencyKey: idempotencyKey, // ← gateway dedup berdasarkan key ini
            UserID:         userID,
            Amount:         amount,
        })
        // Jika request sebelumnya sudah berhasil, gateway return hasil yang sama
        // tanpa memproses ulang — retry 100% aman
    })
}

// Retry transport untuk http.Client — berlaku untuk semua HTTP request
type RetryTransport struct {
    base       http.RoundTripper
    maxRetries int
    baseDelay  time.Duration
}

func (t *RetryTransport) RoundTrip(req *http.Request) (*http.Response, error) {
    retryCount := 0
    for {
        resp, err := t.base.RoundTrip(req)

        if !isRetryable(err, resp) || retryCount >= t.maxRetries {
            return resp, err
        }

        retryCount++
        // Exponential backoff dengan full jitter
        maxDelayMs := int64(t.baseDelay.Milliseconds()) * int64(1<<uint(retryCount))
        jitterMs := rand.Int63n(maxDelayMs)

        log.Printf("[RETRY] attempt %d/%d dalam %dms untuk %s",
            retryCount, t.maxRetries, jitterMs, req.URL.Path)

        time.Sleep(time.Duration(jitterMs) * time.Millisecond)
    }
}

func isRetryable(err error, resp *http.Response) bool {
    // Network error transient
    if err != nil {
        return true
    }
    // HTTP status yang retryable
    return resp != nil && (resp.StatusCode == 429 || resp.StatusCode == 500 ||
        resp.StatusCode == 502 || resp.StatusCode == 503 || resp.StatusCode == 504)
}

// Setup di main
func setupClient() *http.Client {
    return &http.Client{
        Transport: &RetryTransport{maxRetries: 3, baseDelay: 200 * time.Millisecond},
    }
}

// Metrics wajib untuk setiap retry mechanism
func RetryWithMetrics(ctx context.Context, operationName string,
    cfg RetryConfig, fn func() error) error {

    startTime := time.Now()
    totalAttempts := 0

    err := RetryWithBackoff(ctx, cfg, func() error {
        totalAttempts++
        attemptErr := fn()

        if attemptErr != nil && totalAttempts > 1 {
            metrics.Counter("retry.attempt",
                "operation", operationName,
                "attempt", strconv.Itoa(totalAttempts),
            ).Inc()
        }
        return attemptErr
    })

    duration := time.Since(startTime)

    if err != nil {
        // Semua retry habis dan masih gagal
        metrics.Counter("retry.exhausted", "operation", operationName).Inc()
        metrics.Histogram("retry.duration_on_failure",
            "operation", operationName,
        ).Record(duration.Seconds())
    } else if totalAttempts > 1 {
        // Berhasil setelah retry — track berapa attempt
        metrics.Counter("retry.recovered",
            "operation", operationName,
            "attempts", strconv.Itoa(totalAttempts),
        ).Inc()
    }

    return err
}

// ✗ Infinite retry — sistem stuck, resource leak
for {
    err := callService()
    if err == nil { break }
    time.Sleep(100 * time.Millisecond)
}
// ✓ Selalu ada MaxAttempts atau total timeout budget

// ✗ Retry semua error tanpa klasifikasi
RetryWithBackoff(ctx, cfg, func() error {
    return validateUserInput(req) // 400 tidak akan berubah kalau di-retry!
})
// ✓ Definisikan IsRetryable yang eksplisit

// ✗ Retry berlapis tanpa koordinasi — amplification 3×3×3 = 27 calls
// ✓ Tentukan satu layer yang bertanggung jawab, layer lain fail-fast

// ✗ Tidak ada delay — immediate retry untuk network call
for attempt := 0; attempt < 5; attempt++ {
    callExternalService() // flood ke downstream
}
// ✓ Exponential backoff + jitter

// ✗ Retry operasi non-idempotent tanpa idempotency key
RetryWithBackoff(ctx, cfg, func() error {
    return chargeCard(userID, amount) // bisa double charge!
})
// ✓ Pastikan idempotency sebelum menambahkan retry

// ✗ Sembunyikan semua error setelah retry habis
if err := RetryWithBackoff(...); err != nil {
    log.Error(err)
    return nil // ← caller tidak tahu ada masalah!
}
// ✓ Propagate error, biarkan caller memutuskan penanganan selanjutnya

// ✗ Retry setelah context cancelled
for attempt := 1; attempt <= maxAttempts; attempt++ {
    fn()
    time.Sleep(delay) // tidak cek ctx.Done()!
}
// ✓ Selalu cek ctx.Done() di select sebelum tunggu delay