Retry Strategy #

// ANTI-PATTERN: retry without a strategy — dangerous
func callServiceDangerous() error {
    for {                          // ← no limit
        err := callService()
        if err == nil { return nil }
        // no delay — DDoS-ing yourself
        // no error classification — retrying validation errors that will never change
    }
}

// CORRECT: retry with explicit configuration
func callServiceSafe(ctx context.Context) error {
    return RetryWithBackoff(ctx, RetryConfig{
        MaxAttempts:   5,
        BaseDelay:     100 * time.Millisecond,
        MaxDelay:      30 * time.Second,
        Jitter:        true,
        IsRetryable:   isTransientError,
    }, func() error {
        return callService()
    })
}
// 4xx errors → fail fast without retry
// 5xx/timeout errors → retry with increasing delay
// After max attempts → return the final error

// ANTI-PATTERN: immediate retry for network calls
for attempt := 0; attempt < maxAttempts; attempt++ {
    err := callExternalService()
    if err == nil { return nil }
    // ← no delay
    // → when the downstream is down for 100ms, all retries flood it simultaneously
}

// CORRECT: immediate retry only for truly microsecond in-memory operations
for attempt := 0; attempt < 3; attempt++ {
    if acquired := mutex.TryLock(); acquired {
        return nil
    }
    // mutex contention heals in nanoseconds, immediate retry is safe here
    runtime.Gosched() // yield CPU, try again immediately
}

// Fixed delay — simple, fine for simple single-instance systems
func retryFixed(ctx context.Context, maxAttempts int, delay time.Duration,
    fn func() error) error {

    var lastErr error
    for attempt := 1; attempt <= maxAttempts; attempt++ {
        lastErr = fn()
        if lastErr == nil {
            return nil
        }
        if attempt < maxAttempts {
            select {
            case <-time.After(delay):
            case <-ctx.Done():
                return ctx.Err()
            }
        }
    }
    return fmt.Errorf("all %d attempts failed: %w", maxAttempts, lastErr)
}

// ANTI-PATTERN: hundreds of instances retrying with exactly the same delay
// → thundering herd: all 200 instances retry after exactly 1 second
// → the downstream receives a spike of 200 requests at once, not an even spread

// Delay: 100ms → 200ms → 400ms → 800ms → 1600ms (capped)
func calculateBackoff(attempt int, base, max time.Duration) time.Duration {
    delay := base * time.Duration(1<<uint(attempt-1)) // base × 2^(attempt-1)
    if delay > max {
        delay = max
    }
    return delay
}

// Full implementation: exponential backoff + full jitter
func RetryWithBackoff(ctx context.Context, cfg RetryConfig, fn func() error) error {
    var lastErr error

    for attempt := 1; attempt <= cfg.MaxAttempts; attempt++ {
        lastErr = fn()

        if lastErr == nil {
            if attempt > 1 {
                log.Infof("retry succeeded on attempt %d", attempt)
            }
            return nil
        }

        // Fail fast for errors that aren't worth retrying
        if !cfg.IsRetryable(lastErr) {
            return fmt.Errorf("non-retryable error: %w", lastErr)
        }

        if attempt == cfg.MaxAttempts {
            break // don't wait after the last attempt
        }

        // Calculate the base delay: min(base × 2^(attempt-1), maxDelay)
        baseDelay := cfg.BaseDelay * time.Duration(1<<uint(attempt-1))
        if baseDelay > cfg.MaxDelay {
            baseDelay = cfg.MaxDelay
        }

        // Full jitter: random between 0 and baseDelay
        // More effective than ±25% at avoiding thundering herds
        jitter := time.Duration(rand.Int63n(int64(baseDelay)))

        log.Warnf("attempt %d/%d failed: %v — retrying in %v",
            attempt, cfg.MaxAttempts, lastErr, jitter)

        select {
        case <-time.After(jitter):
        case <-ctx.Done():
            return ctx.Err()
        }
    }

    return fmt.Errorf("exhausted %d attempts: %w", cfg.MaxAttempts, lastErr)
}

type RetryConfig struct {
    MaxAttempts int
    BaseDelay   time.Duration
    MaxDelay    time.Duration
    IsRetryable func(error) bool
}

// Retry until the deadline — not until max attempts
func RetryUntilDeadline(ctx context.Context, timeout time.Duration,
    fn func() error) error {

    deadline := time.Now().Add(timeout)
    ctx, cancel := context.WithDeadline(ctx, deadline)
    defer cancel()

    attempt := 0
    baseDelay := 50 * time.Millisecond

    for {
        attempt++
        err := fn()
        if err == nil {
            return nil
        }

        remaining := time.Until(deadline)
        if remaining <= 0 {
            return fmt.Errorf("deadline exceeded after %d attempts: %w", attempt, err)
        }

        nextDelay := baseDelay * time.Duration(1<<uint(attempt-1))
        // Don't retry if the next delay is more than half the remaining time
        if nextDelay > remaining/2 {
            return fmt.Errorf("insufficient time budget for retry: %w", err)
        }

        jitter := time.Duration(rand.Int63n(int64(nextDelay)))
        select {
        case <-time.After(jitter):
        case <-ctx.Done():
            return ctx.Err()
        }
    }
}

// An explicit error classification function
func isRetryableError(err error) bool {
    if err == nil {
        return false
    }

    // Context cancelled or deadline exceeded — don't retry
    if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
        return false
    }

    var httpErr *HTTPError
    if errors.As(err, &httpErr) {
        switch httpErr.StatusCode {
        // ✓ SHOULD retry — transient errors
        case 429: return true // Too Many Requests — wait and try again
        case 500: return true // Internal Server Error — may be transient
        case 502: return true // Bad Gateway — upstream is restarting
        case 503: return true // Service Unavailable — downstream overloaded
        case 504: return true // Gateway Timeout

        // ✗ DON'T retry — deterministic errors
        case 400: return false // Bad Request — the request is wrong, won't change
        case 401: return false // Unauthorized — need to refresh the token first
        case 403: return false // Forbidden — no access
        case 404: return false // Not Found — the resource doesn't exist
        case 409: return false // Conflict — needs special logic, not just retry
        case 422: return false // Unprocessable Entity — invalid data
        }
    }

    // Network errors are transient — usually worth retrying
    var netErr *net.OpError
    if errors.As(err, &netErr) {
        return true
    }

    return false // default: don't retry if unsure
}

// ANTI-PATTERN: retrying a non-idempotent operation
func processPayment(userID string, amount int64) error {
    return RetryWithBackoff(ctx, defaultConfig, func() error {
        // If the request succeeded but the response timed out before arriving,
        // the retry will process a SECOND PAYMENT — double charge!
        return paymentGateway.Charge(userID, amount)
    })
}

// CORRECT: an idempotency key makes retries always safe
func processPayment(idempotencyKey, userID string, amount int64) error {
    return RetryWithBackoff(ctx, defaultConfig, func() error {
        return paymentGateway.Charge(ChargeRequest{
            IdempotencyKey: idempotencyKey, // ← the gateway dedupes based on this key
            UserID:         userID,
            Amount:         amount,
        })
        // If the previous request already succeeded, the gateway returns the same result
        // without reprocessing — retry is 100% safe
    })
}

// Retry transport for http.Client — applies to all HTTP requests
type RetryTransport struct {
    base       http.RoundTripper
    maxRetries int
    baseDelay  time.Duration
}

func (t *RetryTransport) RoundTrip(req *http.Request) (*http.Response, error) {
    retryCount := 0
    for {
        resp, err := t.base.RoundTrip(req)

        if !isRetryable(err, resp) || retryCount >= t.maxRetries {
            return resp, err
        }

        retryCount++
        // Exponential backoff with full jitter
        maxDelayMs := int64(t.baseDelay.Milliseconds()) * int64(1<<uint(retryCount))
        jitterMs := rand.Int63n(maxDelayMs)

        log.Printf("[RETRY] attempt %d/%d in %dms for %s",
            retryCount, t.maxRetries, jitterMs, req.URL.Path)

        time.Sleep(time.Duration(jitterMs) * time.Millisecond)
    }
}

func isRetryable(err error, resp *http.Response) bool {
    // Transient network errors
    if err != nil {
        return true
    }
    // Retryable HTTP statuses
    return resp != nil && (resp.StatusCode == 429 || resp.StatusCode == 500 ||
        resp.StatusCode == 502 || resp.StatusCode == 503 || resp.StatusCode == 504)
}

// Setup in main
func setupClient() *http.Client {
    return &http.Client{
        Transport: &RetryTransport{maxRetries: 3, baseDelay: 200 * time.Millisecond},
    }
}

// Mandatory metrics for every retry mechanism
func RetryWithMetrics(ctx context.Context, operationName string,
    cfg RetryConfig, fn func() error) error {

    startTime := time.Now()
    totalAttempts := 0

    err := RetryWithBackoff(ctx, cfg, func() error {
        totalAttempts++
        attemptErr := fn()

        if attemptErr != nil && totalAttempts > 1 {
            metrics.Counter("retry.attempt",
                "operation", operationName,
                "attempt", strconv.Itoa(totalAttempts),
            ).Inc()
        }
        return attemptErr
    })

    duration := time.Since(startTime)

    if err != nil {
        // All retries exhausted and still failing
        metrics.Counter("retry.exhausted", "operation", operationName).Inc()
        metrics.Histogram("retry.duration_on_failure",
            "operation", operationName,
        ).Record(duration.Seconds())
    } else if totalAttempts > 1 {
        // Recovered after retries — track how many attempts
        metrics.Counter("retry.recovered",
            "operation", operationName,
            "attempts", strconv.Itoa(totalAttempts),
        ).Inc()
    }

    return err
}

// ✗ Infinite retry — system stuck, resource leak
for {
    err := callService()
    if err == nil { break }
    time.Sleep(100 * time.Millisecond)
}
// ✓ Always have MaxAttempts or a total timeout budget

// ✗ Retrying every error without classification
RetryWithBackoff(ctx, cfg, func() error {
    return validateUserInput(req) // a 400 won't change if retried!
})
// ✓ Define an explicit IsRetryable

// ✗ Layered retries without coordination — 3×3×3 = 27 calls amplification
// ✓ Designate one responsible layer, others fail fast

// ✗ No delay — immediate retry for network calls
for attempt := 0; attempt < 5; attempt++ {
    callExternalService() // floods the downstream
}
// ✓ Exponential backoff + jitter

// ✗ Retrying non-idempotent operations without an idempotency key
RetryWithBackoff(ctx, cfg, func() error {
    return chargeCard(userID, amount) // could double charge!
})
// ✓ Ensure idempotency before adding retry

// ✗ Hiding all errors after retries are exhausted
if err := RetryWithBackoff(...); err != nil {
    log.Error(err)
    return nil // ← the caller doesn't know there's a problem!
}
// ✓ Propagate the error, let the caller decide how to handle it

// ✗ Retrying after context cancellation
for attempt := 1; attempt <= maxAttempts; attempt++ {
    fn()
    time.Sleep(delay) // doesn't check ctx.Done()!
}
// ✓ Always check ctx.Done() in a select before waiting for a delay