rm-circle-team-router/suite-split.js

// Circle Suite — Split Tester (level 5).
//
// Level 5 (Apex) is where the Traffic Desk allowance jumps from 20,000 to
// 50,000 impressions a month — the biggest single jump in the ladder. That is
// the first point where a member has enough volume for a comparison between two
// ads to mean anything, so this is the tier where the tool belongs.
//
// What it does: launches 2-3 variants of the same ad AS ONE TEST, splitting the
// impressions evenly, then reads the click counters back and reports which one
// is winning. Nothing here is new plumbing — it creates ads through the same
// Traffic Desk path and reads the same stats — but grouping them into a test
// and doing the arithmetic is the difference between "some ads are running" and
// "this headline beats that one".
//
// The honest part matters most. Display click rates are tiny, so a 3-click
// difference on 400 impressions is noise, not a winner. This module refuses to
// declare a winner until the gap is big enough to be worth acting on, and says
// so plainly instead of showing a confident-looking number that isn't.
'use strict';
const fs = require('fs');
const path = require('path');
const suiteTraffic = require('./suite-traffic');

let DATA_DIR = null;
function init(opts) { DATA_DIR = opts.dataDir; }
function file() { return path.join(DATA_DIR, 'split-tests.json'); }
function readAll() { try { return JSON.parse(fs.readFileSync(file(), 'utf8')); } catch (e) { return {}; } }
function writeAll(v) { try { fs.writeFileSync(file(), JSON.stringify(v)); } catch (e) {} }

const MIN_LEVEL = 5;
const MIN_PER_ARM = 500;      // below this a comparison is meaningless
const MAX_ARMS = 3;

function tests(memberId) {
  const all = readAll();
  return (all[String(memberId)] || []).slice().reverse();
}

function record(memberId, test) {
  const all = readAll();
  const key = String(memberId);
  if (!all[key]) all[key] = [];
  all[key].push(test);
  if (all[key].length > 40) all[key] = all[key].slice(-40);
  writeAll(all);
}

function update(memberId, testId, patch) {
  const all = readAll();
  const rows = all[String(memberId)] || [];
  const t = rows.find(function (r) { return r.testId === testId; });
  if (!t) return null;
  Object.keys(patch).forEach(function (k) { t[k] = patch[k]; });
  writeAll(all);
  return t;
}

// Launch every arm of a test. If an arm fails after others have gone live we
// stop the ones that succeeded, so a member is never left paying for half a
// test they cannot interpret.
async function launch(opts) {
  const level = Number(opts.level) || 1;
  if (level < MIN_LEVEL) throw new Error('The Split Tester unlocks at Apex (level 5).');

  const arms = (Array.isArray(opts.arms) ? opts.arms : []).slice(0, MAX_ARMS);
  if (arms.length < 2) throw new Error('A split test needs at least two versions to compare.');

  const perArm = Math.floor((Number(opts.impressions) || 0) / arms.length);
  if (perArm < MIN_PER_ARM) {
    throw new Error('Give each version at least ' + MIN_PER_ARM.toLocaleString() +
      ' impressions or the result will not mean anything — that is ' +
      (MIN_PER_ARM * arms.length).toLocaleString() + ' total for ' + arms.length + ' versions.');
  }

  const testId = 'st-' + Date.now().toString(36);
  const launched = [];
  try {
    for (let i = 0; i < arms.length; i++) {
      const a = arms[i];
      const entry = await suiteTraffic.launch({
        id: opts.id, level: level, kind: 'text',
        subject: a.subject, lines: a.lines,
        impressions: perArm, target: opts.target, angle: opts.angle,
        hasPage: opts.hasPage,
        name: 'RM Circle #' + opts.id + ' split ' + String.fromCharCode(65 + i)
      });
      launched.push({ arm: String.fromCharCode(65 + i), adId: entry.adId, subject: a.subject, lines: a.lines });
    }
  } catch (err) {
    // Roll back whatever already went live — a half-launched test is worse
    // than no test, and the member gets their impressions back either way.
    for (const l of launched) {
      try { await suiteTraffic.stop(opts.id, l.adId); } catch (e) {}
    }
    throw new Error('Could not launch the whole test, so nothing was left running: ' + (err.message || err));
  }

  const test = {
    testId: testId, at: new Date().toISOString(), perArm: perArm,
    total: perArm * arms.length, arms: launched, stopped: false
  };
  record(opts.id, test);
  return test;
}

// Read live counters and work out where the test stands.
async function results(memberId) {
  const rows = tests(memberId);
  if (!rows.length) return [];
  const ids = [];
  rows.forEach(function (t) { t.arms.forEach(function (a) { ids.push(a.adId); }); });

  let live = [];
  try { live = await suiteTraffic.stats(ids); } catch (e) { live = []; }
  const byId = {};
  live.forEach(function (l) { byId[l.ad_id] = l; });

  return rows.map(function (t) {
    const arms = t.arms.map(function (a) {
      const l = byId[a.adId] || {};
      const served = Math.max(0, Number(l.served) || 0);
      const clicks = Math.max(0, Number(l.hits) || 0);
      return {
        arm: a.arm, adId: a.adId, subject: a.subject, lines: a.lines,
        served: served, clicks: clicks,
        rate: served > 0 ? clicks / served : 0,
        live: !!l.live
      };
    });
    return Object.assign({}, t, { armResults: arms, verdict: verdict(arms) });
  });
}

// Deliberately conservative. Display advertising produces very low click rates,
// so small gaps are noise. We require BOTH arms to have real volume behind them
// and the leader to be clearly ahead before we call anything.
function verdict(arms) {
  const withVolume = arms.filter(function (a) { return a.served >= MIN_PER_ARM / 2; });
  if (withVolume.length < 2) {
    return { state: 'running', text: 'Still gathering impressions — too early to compare.' };
  }
  const sorted = arms.slice().sort(function (x, y) { return y.rate - x.rate; });
  const top = sorted[0], next = sorted[1];
  const totalClicks = arms.reduce(function (s, a) { return s + a.clicks; }, 0);

  if (totalClicks < 10) {
    return { state: 'running', text: 'Only ' + totalClicks + ' clicks so far across all versions — not enough to call it yet.' };
  }
  // Require a 30% relative edge AND at least a few clicks of absolute margin.
  const edge = next.rate > 0 ? (top.rate - next.rate) / next.rate : 1;
  if (edge < 0.3 || (top.clicks - next.clicks) < 3) {
    return { state: 'tie', text: 'No clear winner yet — the versions are performing about the same. That is a real result too: the difference between them is not what is holding the ad back.' };
  }
  return {
    state: 'winner', winner: top.arm,
    text: 'Version ' + top.arm + ' is ahead — ' + top.clicks + ' clicks from ' +
      top.served.toLocaleString() + ' impressions, against ' + next.clicks + ' from ' +
      next.served.toLocaleString() + '. Run the winner on its own next time.'
  };
}

// Stop every arm at once and return the unserved impressions.
async function stop(memberId, testId) {
  const rows = tests(memberId);
  const t = rows.find(function (r) { return r.testId === testId; });
  if (!t) throw new Error('That test is not one of yours.');
  if (t.stopped) throw new Error('That test is already stopped.');
  let refunded = 0;
  for (const a of t.arms) {
    try { const r = await suiteTraffic.stop(memberId, a.adId); refunded += r.refunded || 0; }
    catch (e) { /* already stopped or gone — keep going, the rest still matter */ }
  }
  update(memberId, testId, { stopped: true, stoppedAt: new Date().toISOString(), refunded: refunded });
  return { testId: testId, refunded: refunded };
}

module.exports = { init, launch, results, stop, tests, verdict, MIN_LEVEL, MIN_PER_ARM, MAX_ARMS };