// Circle Suite — Split Tester (level 5). // // Level 5 (Apex) is where the Traffic Desk allowance jumps from 20,000 to // 50,000 impressions a month — the biggest single jump in the ladder. That is // the first point where a member has enough volume for a comparison between two // ads to mean anything, so this is the tier where the tool belongs. // // What it does: launches 2-3 variants of the same ad AS ONE TEST, splitting the // impressions evenly, then reads the click counters back and reports which one // is winning. Nothing here is new plumbing — it creates ads through the same // Traffic Desk path and reads the same stats — but grouping them into a test // and doing the arithmetic is the difference between "some ads are running" and // "this headline beats that one". // // The honest part matters most. Display click rates are tiny, so a 3-click // difference on 400 impressions is noise, not a winner. This module refuses to // declare a winner until the gap is big enough to be worth acting on, and says // so plainly instead of showing a confident-looking number that isn't. 'use strict'; const fs = require('fs'); const path = require('path'); const suiteTraffic = require('./suite-traffic'); let DATA_DIR = null; function init(opts) { DATA_DIR = opts.dataDir; } function file() { return path.join(DATA_DIR, 'split-tests.json'); } function readAll() { try { return JSON.parse(fs.readFileSync(file(), 'utf8')); } catch (e) { return {}; } } function writeAll(v) { try { fs.writeFileSync(file(), JSON.stringify(v)); } catch (e) {} } const MIN_LEVEL = 5; const MIN_PER_ARM = 500; // below this a comparison is meaningless const MAX_ARMS = 3; function tests(memberId) { const all = readAll(); return (all[String(memberId)] || []).slice().reverse(); } function record(memberId, test) { const all = readAll(); const key = String(memberId); if (!all[key]) all[key] = []; all[key].push(test); if (all[key].length > 40) all[key] = all[key].slice(-40); writeAll(all); } function update(memberId, testId, patch) { const all = readAll(); const rows = all[String(memberId)] || []; const t = rows.find(function (r) { return r.testId === testId; }); if (!t) return null; Object.keys(patch).forEach(function (k) { t[k] = patch[k]; }); writeAll(all); return t; } // Launch every arm of a test. If an arm fails after others have gone live we // stop the ones that succeeded, so a member is never left paying for half a // test they cannot interpret. async function launch(opts) { const level = Number(opts.level) || 1; if (level < MIN_LEVEL) throw new Error('The Split Tester unlocks at Apex (level 5).'); const arms = (Array.isArray(opts.arms) ? opts.arms : []).slice(0, MAX_ARMS); if (arms.length < 2) throw new Error('A split test needs at least two versions to compare.'); const perArm = Math.floor((Number(opts.impressions) || 0) / arms.length); if (perArm < MIN_PER_ARM) { throw new Error('Give each version at least ' + MIN_PER_ARM.toLocaleString() + ' impressions or the result will not mean anything — that is ' + (MIN_PER_ARM * arms.length).toLocaleString() + ' total for ' + arms.length + ' versions.'); } const testId = 'st-' + Date.now().toString(36); const launched = []; try { for (let i = 0; i < arms.length; i++) { const a = arms[i]; const entry = await suiteTraffic.launch({ id: opts.id, level: level, kind: 'text', subject: a.subject, lines: a.lines, impressions: perArm, target: opts.target, angle: opts.angle, hasPage: opts.hasPage, name: 'RM Circle #' + opts.id + ' split ' + String.fromCharCode(65 + i) }); launched.push({ arm: String.fromCharCode(65 + i), adId: entry.adId, subject: a.subject, lines: a.lines }); } } catch (err) { // Roll back whatever already went live — a half-launched test is worse // than no test, and the member gets their impressions back either way. for (const l of launched) { try { await suiteTraffic.stop(opts.id, l.adId); } catch (e) {} } throw new Error('Could not launch the whole test, so nothing was left running: ' + (err.message || err)); } const test = { testId: testId, at: new Date().toISOString(), perArm: perArm, total: perArm * arms.length, arms: launched, stopped: false }; record(opts.id, test); return test; } // Read live counters and work out where the test stands. async function results(memberId) { const rows = tests(memberId); if (!rows.length) return []; const ids = []; rows.forEach(function (t) { t.arms.forEach(function (a) { ids.push(a.adId); }); }); let live = []; try { live = await suiteTraffic.stats(ids); } catch (e) { live = []; } const byId = {}; live.forEach(function (l) { byId[l.ad_id] = l; }); return rows.map(function (t) { const arms = t.arms.map(function (a) { const l = byId[a.adId] || {}; const served = Math.max(0, Number(l.served) || 0); const clicks = Math.max(0, Number(l.hits) || 0); return { arm: a.arm, adId: a.adId, subject: a.subject, lines: a.lines, served: served, clicks: clicks, rate: served > 0 ? clicks / served : 0, live: !!l.live }; }); return Object.assign({}, t, { armResults: arms, verdict: verdict(arms) }); }); } // Deliberately conservative. Display advertising produces very low click rates, // so small gaps are noise. We require BOTH arms to have real volume behind them // and the leader to be clearly ahead before we call anything. function verdict(arms) { const withVolume = arms.filter(function (a) { return a.served >= MIN_PER_ARM / 2; }); if (withVolume.length < 2) { return { state: 'running', text: 'Still gathering impressions — too early to compare.' }; } const sorted = arms.slice().sort(function (x, y) { return y.rate - x.rate; }); const top = sorted[0], next = sorted[1]; const totalClicks = arms.reduce(function (s, a) { return s + a.clicks; }, 0); if (totalClicks < 10) { return { state: 'running', text: 'Only ' + totalClicks + ' clicks so far across all versions — not enough to call it yet.' }; } // Require a 30% relative edge AND at least a few clicks of absolute margin. const edge = next.rate > 0 ? (top.rate - next.rate) / next.rate : 1; if (edge < 0.3 || (top.clicks - next.clicks) < 3) { return { state: 'tie', text: 'No clear winner yet — the versions are performing about the same. That is a real result too: the difference between them is not what is holding the ad back.' }; } return { state: 'winner', winner: top.arm, text: 'Version ' + top.arm + ' is ahead — ' + top.clicks + ' clicks from ' + top.served.toLocaleString() + ' impressions, against ' + next.clicks + ' from ' + next.served.toLocaleString() + '. Run the winner on its own next time.' }; } // Stop every arm at once and return the unserved impressions. async function stop(memberId, testId) { const rows = tests(memberId); const t = rows.find(function (r) { return r.testId === testId; }); if (!t) throw new Error('That test is not one of yours.'); if (t.stopped) throw new Error('That test is already stopped.'); let refunded = 0; for (const a of t.arms) { try { const r = await suiteTraffic.stop(memberId, a.adId); refunded += r.refunded || 0; } catch (e) { /* already stopped or gone — keep going, the rest still matter */ } } update(memberId, testId, { stopped: true, stoppedAt: new Date().toISOString(), refunded: refunded }); return { testId: testId, refunded: refunded }; } module.exports = { init, launch, results, stop, tests, verdict, MIN_LEVEL, MIN_PER_ARM, MAX_ARMS };