Running Multi-Region Quorum Checks at Sub-Second Latency

[Target Application / Server]
       ^             ^             ^
       |             |             |  (Parallel pings)
+------+-------------+-------------+------+
| Regional Probe DO  Regional Probe DO    |  (7 Pinned Cloudflare Edge Regions)
| (weur: London)     (enam: Virginia)     |
+------+-------------+-------------+------+
       |             |             |  (Internal Edge RPC)
       v             v             v
+-----------------------------------------+
| Quorum Consensus Engine (Durable Actor) |  (Stateful Quorum Coordinator)
| - Aggregates 4-of-7 consensus           |
| - Manages alert deduplication & flapping|
+--------------------+--------------------+
                     |
                     v
  [Alert Dispatches: Slack / Webhooks / PagerDuty]

// Edge Worker Scheduled Handler
export default {
  async scheduled(
    event: ScheduledEvent,
    env: Env,
    ctx: ExecutionContext,
  ): Promise<void> {
    // 1. Fetch cached target batch assigned to this edge POP
    const targets = await env.DNS_CACHE.get<MonitorTarget[]>(
      "targets:batch:pop",
      "json",
    );

    if (!targets || targets.length === 0) return;

    // 2. Execute parallel pings with strict AbortController timeouts
    const pingPromises = targets.map((target) => executeEdgePing(target, env));

    // 3. Process results without blocking scheduled handler lifecycle
    ctx.waitUntil(Promise.all(pingPromises));
  },
};

async function executeEdgePing(target: MonitorTarget, env: Env): Promise<void> {
  const controller = new AbortController();
  const timeoutId = setTimeout(
    () => controller.abort(),
    target.timeoutMs || 5000,
  );
  const startTime = performance.now();

  try {
    const response = await fetch(target.url, {
      method: target.method || "GET",
      headers: target.headers,
      signal: controller.signal,
    });

    const latencyMs = Math.round(performance.now() - startTime);
    const isSuccess = response.status >= 200 && response.status < 400;

    if (!isSuccess) {
      // Initiate RPC quorum check to Durable Object Coordinator
      await triggerQuorumVote(
        target,
        { success: false, statusCode: response.status, latencyMs },
        env,
      );
    } else {
      // Report success telemetry to state coordinator
      await reportTelemetry(
        target.id,
        { success: true, statusCode: response.status, latencyMs },
        env,
      );
    }
  } catch (err: any) {
    const latencyMs = Math.round(performance.now() - startTime);
    // Timeout or network error — trigger consensus validation
    await triggerQuorumVote(
      target,
      { success: false, error: err.name, latencyMs },
      env,
    );
  } finally {
    clearTimeout(timeoutId);
  }
}

// Durable Object Quorum Coordinator
export class MonitorChannel implements DurableObject {
  private state: DurableObjectState;
  private activeVotes: Map<string, ProbeVote[]> = new Map();

  constructor(state: DurableObjectState, env: Env) {
    this.state = state;
  }

  async fetch(request: Request): Promise<Response> {
    const url = new URL(request.url);

    if (url.pathname === "/vote") {
      const vote: ProbeVote = await request.json();
      return this.handleQuorumVote(vote);
    }

    return new Response("Not Found", { status: 404 });
  }

  private async handleQuorumVote(vote: ProbeVote): Promise<Response> {
    const voteId = vote.batchId;
    const votes = this.activeVotes.get(voteId) || [];
    votes.push(vote);
    this.activeVotes.set(voteId, votes);

    // If this is the first failed vote, immediately request pings from 2 partner POPs
    if (votes.length === 1) {
      await this.requestPartnerProbes(vote.targetUrl, vote.originRegion);

      // Wait up to 1200ms for partner votes to arrive
      await new Promise((resolve) => setTimeout(resolve, 1200));
    }

    const currentVotes = this.activeVotes.get(voteId) || [];
    const failures = currentVotes.filter((v) => !v.success).length;

    // 2 out of 3 (or more) failures = Confirmed Hard Outage
    if (failures >= 2 && !this.isCurrentlyFailing) {
      this.isCurrentlyFailing = true;
      await this.dispatchIncidentAlert(vote.targetId, currentVotes);
    }

    return new Response(JSON.stringify({ confirmed: failures >= 2 }), {
      headers: { "Content-Type": "application/json" },
    });
  }
}