Circuit breaker and graceful degradation when a provider is down

# ======================================================================
# Q: A provider is down for hours. How do you protect the system?
#
# Circuit breaker: after N consecutive failures I stop calling it for a while
# and serve the last good value from cache (graceful degradation: the screen
# shows yesterday's weather instead of nothing). A separate Celery queue per
# integration, so one slow API doesn't fill everyone's workers. Alerts on the
# error rate per provider.
# ======================================================================

def call_with_breaker(name, func, *args, threshold=5, cooldown=120):
    if cache.get(f'breaker:{name}:open'):
        raise ServiceUnavailable(name)
    try:
        result = func(*args)
    except ExternalAPIError:
        cache.get_or_set(f'breaker:{name}:fails', 0, 300)   # initialization with a TTL
        if cache.incr(f'breaker:{name}:fails') >= threshold:
            cache.set(f'breaker:{name}:open', 1, cooldown)
        raise
    cache.delete(f'breaker:{name}:fails')
    return result