Making a Celery task reliable: idempotency, retries and on_commit

# ======================================================================
# Q: How do you make a Celery task reliable?
#
# The task must be idempotent, because with at-least-once delivery it may run
# twice. I pass IDs, not objects. Retries with exponential backoff and jitter
# only for transient errors. acks_late=True so the message is acknowledged after
# the task finishes. Timeouts (soft/hard time limits). I send the task with
# on_commit, so it does not run before the row has been committed to the
# database.
# ======================================================================

@shared_task(bind=True, acks_late=True,
             autoretry_for=(requests.Timeout, requests.ConnectionError),
             retry_backoff=True, retry_jitter=True, max_retries=5)
def sync_order_to_erp(self, order_id):
    order = Order.objects.get(pk=order_id)
    if order.erp_synced_at:          # idempotency: already done
        return
    erp.push(order)                   # the ERP accepts idempotency key = order.uuid
    Order.objects.filter(pk=order_id).update(erp_synced_at=timezone.now())

transaction.on_commit(lambda: sync_order_to_erp.delay(order.id))