Chain feed: four providers, 2000-block logs, a snapshot that cannot stall the tail

The live feed sat silent from 17:58 UTC on 24 September: seven hours of
registrations, upgrades and payouts unannounced. Two faults, one storm.

First, publicnode cut its getLogs range below the 9,000 blocks the tail asked
for ("invalid block range params"), so the tail failed on its own. Second, the
daily snapshot was due. It is ~3,600 calls; it failed part-way on publicnode's
per-IP limit (the site, the IAP app and the cron watchers all share core's one
IP), snapshotAt never advanced, and the next tick started it again from
scratch. Because the snapshot ran BEFORE the tail, every failing attempt also
stopped every live event. 1rpc, the only fallback, had exhausted its free
allowance. The retries were the storm that starved everything else on the IP,
the IAP burner included.

Now: chunk 2000 (measured on publicnode and tenderly), four providers rotated
per call instead of camping on the last one that worked (all four measured
from core; polygon-rpc.com, ankr, blastapi and blockpi refuse), the snapshot
paced at 200 ms per member so it stays under every limit, and a failed
snapshot backs off 30 minutes while the tail runs regardless. A snapshot can
be late; it can no longer make the feed lie by omission.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

chain.js

@@ -14,8 +14,14 @@ const fs = require('fs');
 const path = require('path');
 
 const CONTRACT = '0x33bdaeefd6d17d80ae53816c916dfb26c4fb2daf';
+// Four providers, rotated per call so no single one carries the whole load: the site, the
+// IAP app and the cron watchers all share core's one IP, and on 2026-09-24 publicnode's
+// per-IP limit (1200/min) plus 1rpc's exhausted free allowance stalled this feed for 7 hours.
+// All four measured from core that day; polygon-rpc.com, ankr, blastapi and blockpi refuse.
 const RPCS = [
   'https://polygon-bor-rpc.publicnode.com',
+  'https://polygon.gateway.tenderly.co',
+  'https://polygon.drpc.org',
   'https://1rpc.io/matic'
 ];
 // keccak-256 hashes verified against live logs (tx 0x8a74c439…, 0x45e07f27…)
@@ -32,7 +38,7 @@ const SEL = {
   getMatrixChildren: '0x04c8cc3d',  // getMatrixChildren(uint48)
   getAllCosts: '0x735f87b9'         // getAllCosts()
 };
-const CHUNK = 9000;                 // publicnode getLogs range cap is 10k
+const CHUNK = 2000;                 // measured getLogs span on publicnode and tenderly (9000 now fails: 'invalid block range params')
 const TAIL_MAX_BEHIND = 60000;      // never tail further back than ~1.5 days (log pruning)
 const POLL_MS = 60000;
 const SNAPSHOT_MS = 24 * 3600 * 1000;
@@ -120,8 +126,8 @@ function saveState() {
   } catch (e) { console.error('chain: state save failed', e.message); }
 }
 
-// only publicnode supports ranged getLogs (1rpc caps at 50 blocks)
+// ranged getLogs: publicnode and tenderly take a 2000-block span; drpc and 1rpc cap at 50
-const LOG_RPCS = ['https://polygon-bor-rpc.publicnode.com'];
+const LOG_RPCS = ['https://polygon-bor-rpc.publicnode.com', 'https://polygon.gateway.tenderly.co'];
 let rpcIdx = 0;
 async function rpc(method, params, timeoutMs = 15000, urls = RPCS) {
   let lastErr = new Error('no rpc');
@@ -137,7 +143,7 @@ async function rpc(method, params, timeoutMs = 15000, urls = RPCS) {
       clearTimeout(t);
       const j = await r.json();
       if (j.error) throw new Error(j.error.message || JSON.stringify(j.error));
-      if (urls === RPCS) rpcIdx = (rpcIdx + i) % urls.length;
+      if (urls === RPCS) rpcIdx = (rpcIdx + i + 1) % urls.length;   // round-robin: spread the load, never camp on one provider
       return j.result;
     } catch (e) { lastErr = e; }
   }
@@ -232,7 +238,7 @@ async function snapshot() {
     let inc = [];
     try { inc = await fetchIncome(id); } catch (e) { inc = []; }
     inc.forEach((p, i) => history.push({ key: `h${id}-${i}`, kind: 'income', toId: id, fromId: p.fromId, level: p.level, pol: p.pol, ts: p.ts, desc: describeIncome(p.fromTier, p.level, p.pol) }));
-    await new Promise(r => setTimeout(r, 60));
+    await new Promise(r => setTimeout(r, 200));   // 3 calls per member: ~15/s across four providers, under every limit
   }
   history.sort((a, b) => (a.ts || 0) - (b.ts || 0));
   // keep richer log-sourced entries (they carry tx hashes / upgrade context)
@@ -411,11 +417,23 @@ async function tick() {
   if (busy) return;
   busy = true;
   try {
-    if (!state.snapshotAt || Date.now() - state.snapshotAt > SNAPSHOT_MS) {
+    // The daily snapshot is ~3,600 calls. If it fails part-way it used to be retried from
+    // scratch on the very next tick, and because it ran BEFORE the tail, a failing snapshot
+    // also stopped every live event: that is how the feed sat silent for 7 hours on
+    // 2026-09-24. Now a failure backs off 30 minutes and the tail runs regardless.
+    if ((!state.snapshotAt || Date.now() - state.snapshotAt > SNAPSHOT_MS) && Date.now() >= (state.snapshotRetryAt || 0)) {
+      try {
         const latest = hexInt(await rpc('eth_blockNumber', []));
         await snapshot();
         if (!state.lastBlock) state.lastBlock = latest - 1000; // first run: small log overlap, dedup handles it
+        state.snapshotRetryAt = 0;
         saveState();
+      } catch (e) {
+        state.snapshotRetryAt = Date.now() + 30 * 60 * 1000;
+        saveState();
+        console.error('chain: snapshot failed, next attempt in 30 min; live tail continues:', e.message);
+        if (!state.lastBlock) throw e;   // nothing to tail from yet: this really is a failed tick
+      }
     }
     const latest = hexInt(await rpc('eth_blockNumber', []));
     let from = Math.max(state.lastBlock + 1, latest - TAIL_MAX_BEHIND);