marty/instantadpay
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Screen every advertiser destination against Google Safe Browsing

Browse Source 
Google marked instantadpay.com "deceptive pages" on 24 September. The probable
cause was campaign #225: a verified-visits ad that opened llclickpro.com from
our pages, and llclickpro.com carries exactly that flag. We put other people's
destinations in front of members, so their reputation becomes ours, and
nothing was checking it.

sbcheck.js screens the destination when a campaign is saved (member and house
paths both) and re-screens every live campaign daily, pausing the ones on a
flagged host and alerting the admin channel with the campaigns and owners.
POST /api/admin/sbcheck {dry} runs a pass by hand.

Verdicts are three-valued on purpose: flagged, clean, unknown. A guard that
reads "could not check" as "clean" is the failure mode this whole day has been
about; one that reads it as "flagged" would refuse every advertiser whenever
Google rate-limits us. So flagged refuses and pauses, clean passes, unknown
passes at save time but is logged and retried by the sweep until it resolves.

Uses the official Lookup API v4 when SAFE_BROWSING_KEY is set (full-URL,
batched). Until a key exists it reads the Transparency Report site-status
record, decoded against Google's own test site rather than guessed:
status 1 is clean, status 3 with any true flag is listed, anything else is
no data. That endpoint rate-limits by answering HTML, hence the 4-second pace.

videosweep's inline alert is hoisted into videosweepAlert so both sweeps use
the same admin-only channel: these messages name members and their emails.

QA: member walk 0 bugs, public walk 0 bugs. Verified by hand that
llclickpro.com is refused, rmcircle.team passes, and our own host is skipped.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
martbost
2026-09-24 19:38:35 -05:00
parent f2a999edd0
commit 1b72a9a18c
2 changed files with 3629 additions and 3431 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sbcheck.js
+184
View File
@@ -0,0 +1,184 @@
// Safe Browsing screen for advertiser destinations (Marty, 2026-09-24).
//
// Why this exists: on 24 September Google marked instantadpay.com "some pages on this site are
// unsafe" (deceptive pages). The probable cause was campaign #225, a verified-visits ad that
// opened llclickpro.com from our pages, and llclickpro.com carries exactly that flag. We put
// other people's destinations in front of members, so their reputation becomes ours. This
// screens every destination when a campaign is saved and re-screens the live ones daily.
//
// Two sources, the better one wins:
// 1. Google Safe Browsing Lookup API v4, when SAFE_BROWSING_KEY is set: full-URL, exact,
// batched, no rate-limit trouble. The right tool; needs a Google Cloud API key.
// 2. The Transparency Report site-status endpoint: no key, but site-level only and it
// rate-limits by answering an HTML page instead of JSON. Used until a key exists.
//
// Verdicts are three-valued on purpose: flagged, clean, or UNKNOWN. A guard that reads
// "could not check" as "clean" is the failure mode this whole day has been about; one that
// reads it as "flagged" would refuse every advertiser whenever Google rate-limits us. So:
// flagged refuses at save and pauses in the sweep; clean passes; unknown passes at save time,
// is logged, and is retried by the sweep until it resolves. Nothing is silently treated as fine.
'use strict';
const https = require('https');
let X = null;
const STATE = () => X.path.join(X.dataDir, 'sbcheck.json');
function load() { try { return JSON.parse(X.fs.readFileSync(STATE(), 'utf8')); } catch (e) { return { cache: {}, lastFlagged: [] }; } }
function save(d) { try { X.fs.writeFileSync(STATE(), JSON.stringify(d)); } catch (e) {} }
const CACHE_MS = 12 * 60 * 60 * 1000; // a conclusive verdict is good for half a day; the sweep refreshes live hosts anyway
const THREATS = ['MALWARE', 'SOCIAL_ENGINEERING', 'UNWANTED_SOFTWARE', 'POTENTIALLY_HARMFUL_APPLICATION'];
const PACE_MS = 4000; // spacing between site-status calls; faster than this and Google answers HTML
function init(deps) { X = deps; }
function hostOf(url) { try { return new URL(String(url)).hostname.replace(/^www\./, '').toLowerCase(); } catch (e) { return ''; } }
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
function fetchText(url, opts) {
opts = opts || {};
return new Promise((resolve, reject) => {
const u = new URL(url);
const req = https.request({ hostname: u.hostname, path: u.pathname + u.search, method: opts.method || 'GET',
headers: Object.assign({ 'User-Agent': 'InstantAdPay-sbcheck/1.0' }, opts.headers || {}), timeout: 15000 }, res => {
let body = ''; res.setEncoding('utf8'); res.on('data', d => { body += d; }); res.on('end', () => resolve({ status: res.statusCode, body }));
});
req.on('timeout', () => { req.destroy(new Error('timeout')); });
req.on('error', reject);
if (opts.body) req.write(opts.body);
req.end();
});
}
// --- source 1: the official Lookup API. Returns null when there is no key. ---
async function lookupV4(urls) {
const key = String(process.env.SAFE_BROWSING_KEY || '').trim();
if (!key) return null;
const body = JSON.stringify({ client: { clientId: 'instantadpay', clientVersion: '1.0' },
threatInfo: { threatTypes: THREATS, platformTypes: ['ANY_PLATFORM'], threatEntryTypes: ['URL'], threatEntries: urls.map(u => ({ url: u })) } });
const r = await fetchText('https://safebrowsing.googleapis.com/v4/threatMatches:find?key=' + encodeURIComponent(key),
{ method: 'POST', headers: { 'Content-Type': 'application/json' }, body });
if (r.status !== 200) throw new Error('lookup api http ' + r.status);
const j = JSON.parse(r.body || '{}');
const bad = {};
for (const m of (j.matches || [])) { const u = m.threat && m.threat.url; if (u) bad[u] = m.threatType; }
return urls.map(u => ({ url: u, verdict: bad[u] ? 'flagged' : 'clean', threat: bad[u] || null }));
}
// --- source 2: the Transparency Report, site level, no key. ---
async function siteStatus(host) {
const r = await fetchText('https://transparencyreport.google.com/transparencyreport/api/v3/safebrowsing/status?site=' + encodeURIComponent(host),
{ headers: { 'User-Agent': 'Mozilla/5.0 (compatible; InstantAdPay-sbcheck/1.0)' } });
const line = String(r.body || '').split('\n').find(l => l.startsWith('[["sb.ssr"'));
if (!line) return { verdict: 'unknown', why: 'no answer (rate-limited or the endpoint changed)' };
let arr; try { arr = JSON.parse(line)[0]; } catch (e) { return { verdict: 'unknown', why: 'unparseable answer' }; }
// ["sb.ssr", status, malware, unwantedSoftware, socialEngineering, ?, ?, updatedAt, site, ?]
// status 1 = "No unsafe content found"; 3 with a true flag = "Some pages on this site are unsafe";
// anything else (4, 6...) = no data. Decoded against testsafebrowsing.appspot.com (3,true,true,true)
// and instantadpay.com itself on 2026-09-24 (3,false,false,true = deceptive pages).
const flags = { malware: !!arr[2], 'unwanted software': !!arr[3], 'social engineering': !!arr[4] };
const named = Object.keys(flags).filter(k => flags[k]);
if (named.length) return { verdict: 'flagged', threat: named.join(', ') };
if (arr[1] === 1) return { verdict: 'clean' };
return { verdict: 'unknown', why: 'no data (status ' + arr[1] + ')' };
}
function verdictFor(host, v) {
if (v.verdict === 'flagged') {
return { ok: false, verdict: 'flagged', host, threat: v.threat,
reason: 'Google Safe Browsing currently lists ' + host + ' as unsafe (' + (v.threat || 'harmful content') + '). '
+ 'We cannot send members there: choose a different destination, or get the listing cleared with Google first.' };
}
return { ok: true, verdict: v.verdict, host, why: v.why || null };
}
// Verdict for one destination, used at save time. Cached per host while conclusive.
async function check(url) {
const host = hostOf(url);
if (!host || (X && host === X.selfHost)) return { ok: true, verdict: 'skipped', host };
const st = load();
const c = st.cache[host];
if (c && c.verdict !== 'unknown' && Date.now() - c.at < CACHE_MS) return verdictFor(host, c);
let v;
try {
const v4 = await lookupV4([String(url)]);
v = v4 ? { verdict: v4[0].verdict, threat: v4[0].threat, via: 'lookup' } : Object.assign(await siteStatus(host), { via: 'site-status' });
} catch (e) { v = { verdict: 'unknown', why: e.message, via: 'error' }; }
v.at = Date.now();
st.cache[host] = v; save(st);
if (v.verdict === 'unknown') console.log('sbcheck: could not screen ' + host + ' at save time (' + v.why + '); the sweep will retry');
return verdictFor(host, v);
}
// One pass over every live campaign. Pauses what is flagged, names what could not be checked.
async function run(opts) {
const dry = !!(opts && opts.dry);
if (!X || !X.db || !X.db.enabled()) return { error: 'no database' };
const rows = await X.db.q("SELECT id, type, owner_email, name, target_url, image_url, status FROM campaigns WHERE status='active'");
const byHost = {};
for (const c of rows) for (const u of [c.target_url, c.image_url]) {
const h = hostOf(u); if (!h || h === X.selfHost) continue;
const e = byHost[h] || (byHost[h] = { urls: new Set(), camps: [] });
e.urls.add(String(u)); if (!e.camps.some(x => x.id === c.id)) e.camps.push({ id: c.id, type: c.type, name: c.name, owner: c.owner_email });
}
const hosts = Object.keys(byHost);
const out = { hosts: hosts.length, flagged: [], unknown: [], paused: [], failed: [] };
const st = load();
const verdicts = {};
if (String(process.env.SAFE_BROWSING_KEY || '').trim()) {
try {
const all = []; for (const h of hosts) for (const u of byHost[h].urls) all.push(u);
for (let i = 0; i < all.length; i += 400) {
for (const r of await lookupV4(all.slice(i, i + 400))) {
const h = hostOf(r.url);
if (r.verdict === 'flagged' || !verdicts[h]) verdicts[h] = { verdict: r.verdict, threat: r.threat, via: 'lookup' };
}
}
} catch (e) { for (const h of hosts) verdicts[h] = { verdict: 'unknown', why: e.message, via: 'error' }; }
} else {
for (const h of hosts) {
const c = st.cache[h];
if (c && c.verdict !== 'unknown' && Date.now() - c.at < CACHE_MS) { verdicts[h] = c; continue; }
try { verdicts[h] = Object.assign(await siteStatus(h), { via: 'site-status' }); }
catch (e) { verdicts[h] = { verdict: 'unknown', why: e.message, via: 'error' }; }
await sleep(PACE_MS);
}
}
for (const h of hosts) {
const v = verdicts[h]; v.at = Date.now(); st.cache[h] = v;
if (v.verdict === 'flagged') out.flagged.push({ host: h, threat: v.threat, campaigns: byHost[h].camps });
else if (v.verdict === 'unknown') out.unknown.push({ host: h, why: v.why });
}
if (!dry) {
for (const f of out.flagged) for (const c of f.campaigns) {
try { await X.db.q("UPDATE campaigns SET status='paused' WHERE id=? AND status='active'", [c.id]); out.paused.push(c.id); }
catch (e) {
// never swallow this: the guard exists so a flagged destination cannot keep serving
out.failed.push({ id: c.id, error: e.message });
console.error('sbcheck could not pause #' + c.id + ': ' + e.message);
}
}
st.at = Date.now(); st.lastFlagged = out.flagged.map(f => f.host).sort(); save(st);
if (out.paused.length) {
const lines = out.flagged.map(f => '<b>' + f.host + '</b> (' + f.threat + ')\n'
+ f.campaigns.map(c => ' #' + c.id + ' ' + c.type + ' "' + c.name + '" (' + c.owner + ')').join('\n'));
console.log('sbcheck paused ' + out.paused.length + ' campaign(s) on flagged hosts: ' + out.paused.join(', '));
if (X.alert) {
X.alert('\u{1F6D1} <b>Advertiser destinations Google lists as unsafe</b>\n\n' + lines.join('\n')
+ '\n\nPaused, so our pages stop sending members there and the site’s own Safe Browsing standing is not dragged down again. '
+ 'Credits stay with the advertiser; they can point the campaign somewhere else.').catch(() => {});
}
}
if (out.unknown.length) console.log('sbcheck: ' + out.unknown.length + ' host(s) could not be screened this pass: ' + out.unknown.map(u => u.host + ' (' + u.why + ')').join('; '));
}
return out;
}
// Daily. The first pass runs a couple of minutes after boot so a restart re-screens promptly.
function start() {
const tick = () => { run({}).catch(e => console.error('sbcheck', e.message)); };
setTimeout(tick, 150000);
setInterval(tick, 24 * 60 * 60 * 1000);
}
module.exports = { init, check, run, start, hostOf };
server.js
+17 -3
View File
@@ -36,6 +36,7 @@ const syndicate = require('./syndicate');
const releases = require('./releases');
const ledger = require('./ledger');
const videosweep = require('./videosweep');
const sbcheck = require('./sbcheck'); // Google Safe Browsing screen on advertiser destinations
const updates = require('./updates');
const audit = require('./audit'); // counter audit: views vs delivery logs, charges vs shows (Marty, 2026-09-15) // member update emails from Admin > Releases (Marty, 2026-09-14)
const leaderboard = require('./leaderboard');
@@ -482,8 +483,7 @@ async function boot() {
ledger.init({ dataDir: DATA_DIR });
// re-check live video sources on a schedule: videoCheck only ever runs at save time, so a
// campaign created before it existed, or a link that rots later, is otherwise invisible
videosweep.init({ dataDir: DATA_DIR, fs, path, db, videoCheck,
alert: async text => {
const videosweepAlert = async text => {
const sc = siteConfig();
// Marty's own Hermes chat, where his other operational pings land (@CoolifyHermes_Bot).
// Admin channels ONLY, never a member-visible one: this names a member's campaign and
@@ -493,8 +493,15 @@ async function boot() {
if (sc.telegramBotToken && sc.telegramAdminChatId) return telegramSend(sc.telegramAdminChatId, text);
if (ADMIN_EMAIL && mailer.hasKey()) return mailer.send(ADMIN_EMAIL, 'InstantAdPay: broken video sources', text.replace(/<[^>]+>/g, ''));
console.log('videosweep: no admin channel configured, alert not sent');
} });
};
videosweep.init({ dataDir: DATA_DIR, fs, path, db, videoCheck, alert: videosweepAlert });
videosweep.start();
// Screen every advertiser destination against Google Safe Browsing: refuse a flagged one at
// save time, and re-screen the live ones daily. Added 2026-09-24 after Google marked the
// site "deceptive pages" with campaign #225 opening llclickpro.com (flagged) from our pages.
// Shares videosweep's alert path on purpose: it names members and their emails, admin only.
sbcheck.init({ dataDir: DATA_DIR, fs, path, db, selfHost: 'instantadpay.com', alert: videosweepAlert });
sbcheck.start();
toolkit.init({ dataDir: DATA_DIR, ads, accounts, siteConfig, coach, messages, promos, videomaker, chain });
updates.init({ dataDir: DATA_DIR, accounts, releases, mailer, drip, sendy, adminEmail: ADMIN_EMAIL });
audit.init({ dataDir: DATA_DIR, notify: text => { const sc = siteConfig(); if (sc.telegramBotToken && sc.telegramAdminChatId) telegramSend(sc.telegramAdminChatId, text).catch(() => {}); else if (ADMIN_EMAIL && mailer.hasKey()) mailer.send(ADMIN_EMAIL, 'InstantAdPay: counter audit', text).catch(() => {}); } });
@@ -2923,6 +2930,7 @@ const server = http.createServer(async (req, res) => {
if (!fc.ok) return json(res, 400, { error: fc.reason });
}
if (String(b.type) === 'video') { const vc = await videoCheck(b.videoUrl); if (!vc.ok) return json(res, 400, { error: vc.reason }); }
{ const sb = await sbcheck.check(b.targetUrl); if (!sb.ok) return json(res, 400, { error: sb.reason }); }
// charge the best-funded of the member's positions (main + Qualified Start
// wallets). A campaign burns from one member id, so the budget must fit inside it.
const ids = (await myMemberIds(s)).ids;
@@ -3099,6 +3107,7 @@ const server = http.createServer(async (req, res) => {
// A house video was never checked: videoCheck guarded the member path only, so a bad
// source pasted here would hand every viewer a black player exactly as #146 did.
if (String(b.type) === 'video') { const vc = await videoCheck(b.videoUrl); if (!vc.ok) return json(res, 400, { error: vc.reason }); }
{ const sb = await sbcheck.check(b.targetUrl); if (!sb.ok) return json(res, 400, { error: sb.reason }); }
if (!['login', 'solo', 'video', 'featured'].includes(String(b.type || ''))) {
if (b.type === 'banner' || (b.type === 'login' && b.imageUrl)) { const ic = await imageCheck(b.imageUrl); if (!ic.ok) return json(res, 400, { error: ic.reason }); }
const fc = await frameCheck(b.targetUrl);
@@ -3263,6 +3272,11 @@ const server = http.createServer(async (req, res) => {
const b = await readBody(req);
return json(res, 200, await videosweep.run({ dry: !!b.dry }));
}
if (p === '/api/admin/sbcheck' && req.method === 'POST') {
if (!isAdmin(req)) return json(res, 401, { error: 'auth' });
const b = await readBody(req);
return json(res, 200, await sbcheck.run({ dry: !!b.dry }));
}
if (p === '/api/admin/burner' && req.method === 'GET') {
if (!isAdmin(req)) return json(res, 401, { error: 'auth' });
return json(res, 200, burner.status());