.NET Microservices, Performance & Docker

# ======================================================================
# Q: An endpoint is slow in production. How do you investigate?
#
# Measure first, don't guess. I start with the traces (Application Insights /
# OpenTelemetry) to see where the time goes: database, external HTTP, CPU, or
# waiting for a thread. I look at p95/p99, not the average.
# - Many small DB queries in the trace: N+1 (Q7).
# - One slow query on a large table: missing index, OFFSET pagination (Q9).
# - Latency rises sharply with load while CPU stays low: thread pool starvation
#   from sync-over-async (Q4).
# - Many sequential HTTP calls: chatty integration, missing caching or
#   Task.WhenAll (Q43).
# - High GC, memory keeps growing: allocations, buffering of large files, a
#   memory leak in a static/singleton (Q44).
# - Load testing before release: k6, NBomber or Azure Load Testing, with
#   realistic data.
# - Micro-benchmarks with BenchmarkDotNet, always in Release.
# - Set a target (SLO, e.g. p95 < 300 ms) so you know when to stop optimizing.
# ======================================================================

# Live metrics from the process: thread pool queue, GC, exceptions, requests/sec
dotnet tool install -g dotnet-counters
dotnet-counters monitor -p <pid> --counters System.Runtime,Microsoft.AspNetCore.Hosting

dotnet-trace collect -p <pid>     # CPU profile: opens in PerfView / Visual Studio / speedscope
dotnet-dump collect -p <pid>      # memory dump: for hangs and memory leaks

// ======================================================================
// Q: How do you debug and operate a distributed system?
//
// - Distributed tracing with OpenTelemetry: the trace context (W3C traceparent)
//   flows automatically through HTTP calls and Service Bus messages, so you can
//   see the whole path of an order across all services.
// - Structured logs with OrderId and MessageId as fields, not inside the message
//   text.
// - Metrics and alerts: queue length, age of the oldest message, DLQ size,
//   consumer lag.
// - Health checks (/health/live, /health/ready) for Kubernetes / Container Apps.
// - Versioning of message contracts: additive changes only (new optional
//   fields). The consumer is a tolerant reader (ignores unknown fields). For a
//   breaking change: a new message type (OrderPlacedV2), and for a while you
//   publish both.
// - API Gateway (YARP, Azure API Management): a single entry point for auth,
//   rate limiting and routing. It contains no business logic.
// - .NET Aspire: spins up all services, the database and the broker locally,
//   with service discovery and a dashboard for traces and logs.
// A good closing line: "Microservices solve a team-organization and scaling
// problem, not a code problem. If the code is tangled in a monolith, it will be
// tangled when distributed too."
// ======================================================================

// OpenTelemetry (Azure Monitor) and health checks
builder.Services.AddOpenTelemetry()
    .UseAzureMonitor()                                   // Application Insights
    .WithTracing(t => t.AddSource("Shop.Ordering"))      // your own ActivitySource
    .WithMetrics(m => m.AddMeter("Shop.Ordering"));

builder.Services.AddHealthChecks()
    .AddDbContextCheck<AppDbContext>(tags: ["ready"]);

// ======================================================================
// Q: Order, stock and payment are in different services. How do you keep them
// consistent?
//
// A distributed transaction (2PC) is not an option. I use a Saga: a sequence of
// local transactions, where each step has a compensating action in case a later
// step fails.
// - Choreography: each service listens to events and publishes its own. Pros:
//   simple for 2-3 steps, no central point. Cons: the flow is "hidden" across
//   many services, with a risk of cyclic events.
// - Orchestration: an orchestrator (state machine) sends commands and keeps the
//   state. Pros: the flow is visible in one place, timeouts and compensations
//   are easy. Cons: the orchestrator can turn into a "god service".
// Tools: MassTransit state machines, NServiceBus sagas, Azure Durable Functions,
// Temporal. The user sees intermediate states (e.g. "Pending payment"): that is
// eventual consistency, and it must be visible in the UI too.
// ======================================================================

// Orchestration: the saga as an aggregate with state, persisted in the database
public class OrderFulfillmentSaga
{
    public Guid OrderId { get; private set; }
    public SagaState State { get; private set; } = SagaState.AwaitingStock;

    public IEnumerable<object> On(StockReserved e)
    {
        State = SagaState.AwaitingPayment;
        yield return new ChargePayment(OrderId, e.Amount);            // next command
    }

    public IEnumerable<object> On(PaymentFailed e)
    {
        State = SagaState.Compensating;
        yield return new ReleaseStock(OrderId);                       // compensation
        yield return new CancelOrder(OrderId, e.Reason);
    }
}
// Commands go out via the outbox (Q10), so state and messages are saved together

# ======================================================================
# Q: Order, stock and payment are in different services. How do you keep them
# consistent?
#
# A distributed transaction (2PC) is not an option. I use a Saga: a sequence of
# local transactions, where each step has a compensating action in case a later
# step fails.
# - Choreography: each service listens to events and publishes its own. Pros:
#   simple for 2-3 steps, no central point. Cons: the flow is "hidden" across
#   many services, with a risk of cyclic events.
# - Orchestration: an orchestrator (state machine) sends commands and keeps the
#   state. Pros: the flow is visible in one place, timeouts and compensations
#   are easy. Cons: the orchestrator can turn into a "god service".
# Tools: MassTransit state machines, NServiceBus sagas, Azure Durable Functions,
# Temporal. The user sees intermediate states (e.g. "Pending payment"): that is
# eventual consistency, and it must be visible in the UI too.
# ======================================================================

OrderPlaced --> [Stock] ReserveStock --> StockReserved --> [Payment] Charge
                                                                  |
                                                            PaymentFailed
                                                                  |
                         compensation:  [Stock] ReleaseStock  <---+
                                        [Ordering] CancelOrder <--+

// ======================================================================
// Q: A message is delivered twice. Or out of order. What happens?
//
// Brokers provide at-least-once delivery: "exactly-once" doesn't really exist in
// practice. So the consumer must be idempotent.
// - Natural idempotency where possible: "set status = Shipped" or an upsert,
//   instead of "add 1".
// - Ordering: the queue doesn't guarantee order when there are multiple
//   consumers or retries. Solutions: Service Bus sessions (FIFO per SessionId =
//   OrderId), a Kafka partition key, or a version / sequence number in the
//   message, where the consumer ignores anything older than what it already has.
// - Dead-letter queue: alert when it isn't empty, a tool for inspection and
//   replay, and a human who looks at it.
// - Keep messages small: IDs and the necessary fields, not whole entities. For
//   large payloads: the claim-check pattern (blob storage plus a link in the
//   message).
// ======================================================================

// Idempotent consumer with an inbox table:
// unique index (MessageId, Consumer), in the same transaction as the change
public async Task HandleAsync(OrderPlaced e, string messageId, CancellationToken ct)
{
    if (await db.ProcessedMessages.AnyAsync(m => m.MessageId == messageId && m.Consumer == "erp-sync", ct))
        return;                                                  // already processed

    await using var tx = await db.Database.BeginTransactionAsync(ct);
    db.ProcessedMessages.Add(new ProcessedMessage(messageId, "erp-sync", DateTime.UtcNow));
    db.ErpExports.Add(new ErpExport(e.OrderId, ErpExportStatus.Pending));

    try
    {
        await db.SaveChangesAsync(ct);
        await tx.CommitAsync(ct);
    }
    catch (DbUpdateException ex) when (ex.IsUniqueViolation())   // our own extension (e.g. PostgreSQL 23505)
    {
        // two instances got the same message concurrently: the unique index keeps only one
    }
}