martbost
f27388a322
Follow-on to the risk rewrite. Two gaps it exposed. Members ask this without the word PIF: "should I fund their first package", "can I buy it for them", "can I front someone the $20". Those were falling through to the AI, which answered reasonably but told the member to "read the full warning on your training page" - a page section that does not exist and that they would go looking for. The pattern now also matches a paying verb plus an explicit for-someone-else phrase, so these get the full canned answer with the risk in it. It deliberately needs BOTH halves: "which package should I buy" must not be stolen. The prompt now forbids sending anyone to a warning, guide or page section that is not in the PAGES list, and requires plain ASCII (the model was emitting non-breaking hyphens in "Wi-Fi"). Fixed two first-match-wins routing bugs, one of them mine from the previous commit: "what do I get for buying a package" was returning the commission split, because I had widened the earnings pattern to "what do I get" when it should only ever have been "what do I get paid". And "should I buy the $20 package" previously matched nothing at all and burned an AI call on a question we have a written answer for. CANNED is an ordered list, so any pattern edit can silently steal a neighbour. The suite now pins a 19-case routing table across PIF, price, earnings and the tank, so the next person to widen a regex finds out immediately. qa/chatbot-parse.mjs is 35 assertions. qa/run.sh member: 0 bugs. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
104 lines
5.6 KiB
JavaScript
104 lines
5.6 KiB
JavaScript
// The OpenRouter response shapes that broke the chatbot on 2026-09-17, and the ones
|
|
// that must keep working. Runs the real parse branch from chatbot.js with a fake https
|
|
// server, so this is the actual code path, not a copy of it.
|
|
//
|
|
// Run: node qa/chatbot-parse.mjs
|
|
import http from 'node:http';
|
|
import https from 'node:https';
|
|
import { createRequire } from 'node:module';
|
|
const require = createRequire(import.meta.url);
|
|
|
|
const ok = [], bad = [];
|
|
const t = (n, c, extra) => { (c ? ok : bad).push(n + (c || !extra ? '' : ' -> ' + extra)); };
|
|
|
|
let BODY = '';
|
|
const srv = http.createServer((req, res) => { res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(BODY); });
|
|
await new Promise(r => srv.listen(0, '127.0.0.1', r));
|
|
const port = srv.address().port;
|
|
|
|
// point chatbot.js's https.request at the local plain-http stub
|
|
const realRequest = https.request;
|
|
https.request = (opts, cb) => http.request({ ...opts, hostname: '127.0.0.1', port, protocol: 'http:', rejectUnauthorized: false }, cb);
|
|
process.env.OPENROUTER_API_KEY = 'test-key';
|
|
const cb = require('../chatbot.js');
|
|
cb.init({ dataDir: '.', chain: null });
|
|
|
|
const good = JSON.stringify({ choices: [{ message: { content: 'Here is a straight answer.' } }] });
|
|
const ask = async q => (await cb.answer(q)).reply;
|
|
const FAILMSG = /did not come back to me just now/;
|
|
|
|
// 1. the exact production failure: 200 with only keepalive padding, no JSON at all
|
|
BODY = '\n \n \n';
|
|
t('keepalive-only body no longer crashes into a dead end', FAILMSG.test(await ask('explain something obscure')));
|
|
|
|
// 2. the shape that ALSO arrives normally: keepalive padding, THEN the real JSON
|
|
BODY = '\n \n' + good;
|
|
t('keepalive padding before real JSON still parses', /straight answer/.test(await ask('explain something obscure')));
|
|
|
|
// 3. plain JSON, the happy path
|
|
BODY = good;
|
|
t('plain JSON still parses', /straight answer/.test(await ask('explain something obscure')));
|
|
|
|
// 4. SSE-style comment lines before the JSON
|
|
BODY = ': OPENROUTER PROCESSING\n: OPENROUTER PROCESSING\n' + good;
|
|
t('SSE comment lines before the JSON still parse', /straight answer/.test(await ask('explain something obscure')));
|
|
|
|
// 5. 200 with an empty content string is a failure, not a blank reply to the member
|
|
BODY = JSON.stringify({ choices: [{ message: { content: ' ' } }] });
|
|
t('empty content is treated as a failure, never sent as a blank answer', FAILMSG.test(await ask('explain something obscure')));
|
|
|
|
// 6. garbage that is not JSON at all
|
|
BODY = '<html>502 bad gateway</html>';
|
|
t('non-JSON body is handled', FAILMSG.test(await ask('explain something obscure')));
|
|
|
|
// 7. canned answers never touch the network, so they survive any outage
|
|
BODY = '';
|
|
const pif = await ask('Should I pif someone from the holding tank');
|
|
t('the PIF question answers with no AI at all', /PIF is a gift/.test(pif), pif.slice(0, 80));
|
|
t('and it says you cannot PIF someone still in the tank', /still sitting in the holding tank/.test(pif));
|
|
// Marty, 2026-09-17: the risk is the answer, not a footnote. Sending POL to someone
|
|
// who has never replied is the easiest money on the site to lose, so every one of
|
|
// these must survive any future edit to the wording.
|
|
t('it says talk to them and get a commitment FIRST', /do not send anything until you have actually talked to that person/i.test(pif));
|
|
t('it says the transfer is irreversible with no refund', /no refund, no chargeback/i.test(pif));
|
|
t('it says support cannot pull it back', /nothing we can do to pull it back/i.test(pif));
|
|
t('it says they may go quiet or spend it elsewhere', /go quiet, spend it on something else, or never sign in again/i.test(pif));
|
|
t('it flags the tank member as the highest risk case', /easiest money on this site to lose/i.test(pif));
|
|
t('it gives the honest arithmetic both ways', /costs you about half/.test(pif) && /costs you all of it/.test(pif));
|
|
t('it says never gift money you need and never borrow', /never gift money you need, never borrow/i.test(pif));
|
|
t('it never sells PIF as a tactic', !/you should (definitely |really )?pif/i.test(pif));
|
|
|
|
// 8. ROUTING. CANNED is first-match-wins, so widening any pattern can silently steal a
|
|
// neighbour. Both of these already happened: "how much do I earn" returned the price
|
|
// ladder, and "what do I get for buying a package" returned the commission split. Every
|
|
// pattern edit must keep this table green.
|
|
const WANT = {
|
|
PIF: /PIF is a gift/,
|
|
PRICE: /Five packages, priced/,
|
|
EARN: /Every package splits the same way/,
|
|
TANK: /The holding tank is where free members/
|
|
};
|
|
const ROUTES = [
|
|
['PIF', 'should I pif someone from the holding tank'], ['PIF', 'what is pif'],
|
|
['PIF', 'should i pay it forward'], ['PIF', 'can I front someone the $20'],
|
|
['PIF', 'should I spot them the money'], ['PIF', 'should I pay for their first package'],
|
|
['PIF', 'can I just buy the package for someone in my downline who wont respond'],
|
|
['PIF', 'is it smart to fund a new members first package for them'],
|
|
['PRICE', 'which package should i buy'], ['PRICE', 'how much is the starter package'],
|
|
['PRICE', 'should I buy the $20 package'], ['PRICE', 'what do I get for buying a package'],
|
|
['PRICE', 'how much does it cost'],
|
|
['EARN', 'how much do I earn'], ['EARN', 'how much can i make'],
|
|
['EARN', 'what will i get paid'], ['EARN', 'what is the commission'],
|
|
['TANK', 'how does the holding tank work'], ['TANK', 'can I adopt someone from the tank']
|
|
];
|
|
for (const [want, q] of ROUTES) {
|
|
const reply = await ask(q);
|
|
t('routes to ' + want + ': "' + q + '"', WANT[want].test(reply), String(reply).slice(0, 55));
|
|
}
|
|
|
|
https.request = realRequest;
|
|
srv.close();
|
|
console.log('PASS ' + ok.length);
|
|
for (const b of bad) console.log('FAIL ' + b);
|
|
process.exit(bad.length ? 1 : 0);
|