A/B and Multivariate Testing

The previous lesson found the weakest step in the borrow flow: the transition to borrow confirmation, at 55.2 percent. The next task is to try a change. But the measured rate coming out different after the change does not mean the change worked — funnel rates move from week to week even when nothing changes.

An A/B test is an experiment that randomly splits users into two groups, shows one the current interface and the other the changed interface, and measures the difference. This lesson takes up four questions about the experiment: how many users are needed, when a measured difference is real, why the experiment is not stopped early, and how several changes are tested at the same time?

Four Computations

// experiment.mjs — sample size, confidence interval, early stopping, and multiple comparisons

function generator(seed) {
  let s = seed >>> 0;
  return () => { s = (s * 1664525 + 1013904223) >>> 0; return s / 4294967296; };
}

// Normal distribution functions (Abramowitz-Stegun approximation + bisection for the inverse)
function normalCdf(x) {
  const t = 1 / (1 + 0.2316419 * Math.abs(x));
  const d = 0.3989422804014327 * Math.exp(-x * x / 2);
  const p = d * t * (0.319381530 + t * (-0.356563782 + t * (1.781477937 + t * (-1.821255978 + t * 1.330274429))));
  return x >= 0 ? 1 - p : p;
}
function z(p) {
  let low = -10, high = 10;
  for (let i = 0; i < 200; i++) {
    const mid = (low + high) / 2;
    if (normalCdf(mid) < p) low = mid; else high = mid;
  }
  return (low + high) / 2;
}

// ---- 1) Sample size: to detect a difference between two ratios ----
// From the funnel lesson: the current rate of the borrow confirmation step is 0.552.
const BASELINE = 0.552;
function sampleSize(p1, p2, alpha = 0.05, power = 0.80) {
  const zAlpha = z(1 - alpha / 2), zBeta = z(power);
  const mean = (p1 + p2) / 2;
  const numerator = zAlpha * Math.sqrt(2 * mean * (1 - mean)) + zBeta * Math.sqrt(p1 * (1 - p1) + p2 * (1 - p2));
  return Math.ceil((numerator * numerator) / ((p1 - p2) ** 2));
}
console.log("detectable difference  target rate  users per group  total");
for (const diff of [0.01, 0.02, 0.03, 0.05, 0.08]) {
  const n = sampleSize(BASELINE, BASELINE + diff);
  console.log(
    `${((diff * 100).toFixed(0) + " pts").padStart(22)} ${((BASELINE + diff).toFixed(3)).padStart(11)}` +
    ` ${String(n).padStart(22)} ${String(2 * n).padStart(7)}`
  );
}
console.log(`(baseline rate ${BASELINE}, alpha 0.05 two-tailed, power 0.80)`);

// ---- 2) Confidence interval of the rate difference ----
console.log("\ngroup A         group B         diff      95% confidence interval    result");
const EXPERIMENTS = [
  { nA: 2000, kA: 1104, nB: 2000, kB: 1160 },
  { nA: 2000, kA: 1104, nB: 2000, kB: 1260 },
  { nA: 8000, kA: 4416, nB: 8000, kB: 4640 },
];
for (const d of EXPERIMENTS) {
  const pA = d.kA / d.nA, pB = d.kB / d.nB, diff = pB - pA;
  const se = Math.sqrt(pA * (1 - pA) / d.nA + pB * (1 - pB) / d.nB);
  const [low, high] = [diff - 1.96 * se, diff + 1.96 * se];
  console.log(
    `${(d.kA + "/" + d.nA).padEnd(15)} ${(d.kB + "/" + d.nB).padEnd(15)} ${((diff * 100).toFixed(2) + " pts").padStart(8)}` +
    `  [${(low * 100).toFixed(2)}, ${(high * 100).toFixed(2)}] pts   ${low > 0 || high < 0 ? "difference found" : "inconclusive"}`
  );
}

// ---- 3) Early stopping: how often is "difference found" said when there is truly no difference ----
const RUNS = 2000, GROUP_SIZE = 8000, CHECK_INTERVAL = 500;
function aaTrial(rnd, peeking) {
  let kA = 0, kB = 0, signaled = false;
  for (let i = 1; i <= GROUP_SIZE; i++) {
    if (rnd() < BASELINE) kA++;
    if (rnd() < BASELINE) kB++;
    if (peeking && i % CHECK_INTERVAL === 0 && i >= CHECK_INTERVAL * 2) {
      if (Math.abs(zValue(kA, i, kB, i)) > 1.96) { signaled = true; break; }
    }
  }
  if (!peeking) signaled = Math.abs(zValue(kA, GROUP_SIZE, kB, GROUP_SIZE)) > 1.96;
  return signaled;
}
function zValue(kA, nA, kB, nB) {
  const pA = kA / nA, pB = kB / nB, mean = (kA + kB) / (nA + nB);
  const se = Math.sqrt(mean * (1 - mean) * (1 / nA + 1 / nB));
  return se === 0 ? 0 : (pB - pA) / se;
}
console.log("\nA/A experiment (both groups see the same interface, true difference is zero)");
for (const peeking of [false, true]) {
  const rnd = generator(20260909);
  let flagged = 0;
  for (let k = 0; k < RUNS; k++) if (aaTrial(rnd, peeking)) flagged++;
  console.log(
    `${(peeking ? `peeking every ${CHECK_INTERVAL} users` : "looking only at the end").padEnd(32)}` +
    ` false "difference found": ${(flagged / RUNS * 100).toFixed(1)} % (${flagged}/${RUNS})`
  );
}

// ---- 4) Multiple comparisons: family-wise error in a multivariate test ----
console.log("\ncomparisons  uncorrected family error  Bonferroni alpha  Sidak alpha");
for (const k of [1, 3, 7, 15]) {
  const familyError = 1 - Math.pow(0.95, k);
  console.log(
    `${String(k).padStart(11)} ${((familyError * 100).toFixed(1) + " %").padStart(25)}` +
    ` ${(0.05 / k).toFixed(4).padStart(16)} ${(1 - Math.pow(0.95, 1 / k)).toFixed(4).padStart(11)}`
  );
}

// 2x2x2 multivariate test: 8 combinations, 7 comparisons against the control
const K = 7, CORRECTED_ALPHA = 0.05 / K;
console.log(`\n2x2x2 multivariate test: 8 combinations, ${K} comparisons`);
console.log(`uncorrected family error ${((1 - Math.pow(0.95, K)) * 100).toFixed(1)} %, Bonferroni alpha ${CORRECTED_ALPHA.toFixed(4)}`);
console.log(`users per group to detect a 2-point difference with the corrected alpha: ${sampleSize(BASELINE, BASELINE + 0.02, CORRECTED_ALPHA)}`);
console.log(`total for 8 combinations: ${8 * sampleSize(BASELINE, BASELINE + 0.02, CORRECTED_ALPHA)} users`);
console.log(`total in an uncorrected, single-comparison A/B test: ${2 * sampleSize(BASELINE, BASELINE + 0.02)} users`);

detectable difference  target rate  users per group  total
                 1 pts       0.562                  38734   77468
                 2 pts       0.572                   9660   19320
                 3 pts       0.582                   4282    8564
                 5 pts       0.602                   1532    3064
                 8 pts       0.632                    592    1184
(baseline rate 0.552, alpha 0.05 two-tailed, power 0.80)

group A         group B         diff      95% confidence interval    result
1104/2000       1160/2000       2.80 pts  [-0.27, 5.87] pts   inconclusive
1104/2000       1260/2000       7.80 pts  [4.76, 10.84] pts   difference found
4416/8000       4640/8000       2.80 pts  [1.26, 4.34] pts   difference found

A/A experiment (both groups see the same interface, true difference is zero)
looking only at the end          false "difference found": 5.7 % (114/2000)
peeking every 500 users          false "difference found": 20.4 % (408/2000)

comparisons  uncorrected family error  Bonferroni alpha  Sidak alpha
          1                     5.0 %           0.0500      0.0500
          3                    14.3 %           0.0167      0.0170
          7                    30.2 %           0.0071      0.0073
         15                    53.7 %           0.0033      0.0034

2x2x2 multivariate test: 8 combinations, 7 comparisons
uncorrected family error 30.2 %, Bonferroni alpha 0.0071
users per group to detect a 2-point difference with the corrected alpha: 15351
total for 8 combinations: 122808 users
total in an uncorrected, single-comparison A/B test: 19320 users

Sample Size Is Computed Before the Experiment

The first table gives the experiment’s most frequently skipped step: how many users are needed is computed before the users arrive.

The computation needs four inputs. The baseline rate (0.552, from the funnel lesson), the smallest difference to be detected, the false-positive rate $\alpha$ (0.05), and the power (0.80 — the probability of catching a real difference if one exists).

The real information the table carries is a scaling relationship: the required sample size is inversely proportional to the square of the detectable difference. Catching an eight-point difference needs 592 users per group; catching a one-point difference needs 38734. Cutting the difference to an eighth multiplies the sample by sixty-five.

The practical consequence is that the experiment is expensive for small improvements. If the catalog interface sees five thousand borrow confirmations a week, measuring a two-point difference takes about four weeks; measuring a one-point difference takes fifteen. The experiment’s duration depends on the size of the change being tested, and if that duration is not available, the experiment is not run — the change is defended on some other basis, or not at all.

The Same Difference, Two Different Outcomes

The second table shows what the question “is there a difference” depends on. The measured difference in the first and third rows is exactly the same: 2.80 points. In the first row, with groups of 2000, the confidence interval is [-0.27, 5.87] and includes zero; the result is inconclusive. In the third row, with groups of 8000, the interval is [1.26, 4.34] and does not include zero; there is a difference.

The only difference between them is the sample size. The sentence “we measured a 2.8 percent improvement” says nothing without the sample size written alongside it.

The second row gives the opposite case: a large difference of 7.80 points stays outside the interval even with groups of 2000. Large effects can be detected with a small sample; the same relationship as in the first table.

The width of the confidence interval also carries decision information. In the third row, the true difference lies between 1.26 and 4.34 points; if the change’s cost is only covered by a 4-point gain, the interval is not enough to make that call. “Statistically significant” and “large enough to change the decision” are not the same thing.

Early Stopping Inflates the False-Positive Rate

The third computation sets up an A/A experiment: both groups see the same interface, and the true difference is zero. A correctly working test should say “difference found” by mistake in 5 percent of cases here.

Looking only at the end of the experiment, the rate comes out at 5.7 percent — close to the expected 5 percent; the difference is the simulation fluctuation of two thousand runs. When the test peeks every 500 users and stops as soon as the boundary is crossed, the rate rises to 20.4 percent. With no true difference at all, one in five experiments says “the change worked.”

The reason is that the test carries a 5 percent margin of error at every look. Fourteen separate looks are fourteen separate chances to be wrong. A randomly fluctuating difference sooner or later crosses the boundary once, and if the test stops at that moment, the value that crossed it is recorded as the result.

Rule: the experiment is not stopped, and no decision is made, until it reaches the computed sample size. Looking at the numbers while the experiment is running is not forbidden; looking and then stopping is. In situations where an interim decision is genuinely needed, sequential-testing methods that tighten the boundary according to the number of interim looks are used; an interim decision is not made with the ordinary boundary.

The Cost of a Multivariate Test

The fourth table gives another form of the same problem. When more than one comparison is made in an experiment, the probability of being wrong in at least one of them — the family-wise error — rises together with the number of comparisons. 30.2 percent at seven comparisons, 53.7 percent at fifteen.

A multivariate test changes more than one element in the same experiment and compares their combinations. Three elements with two values each produce $2^3 = 8$ combinations, and seven comparisons against the control. Without correction, this means the experiment has a 30.2 percent probability of reporting at least one false “winner.”

Correction tightens each comparison’s boundary. The Bonferroni correction divides $\alpha$ by the number of comparisons (0.0071); the Šidák correction fixes the family error directly at 0.05 (0.0073) and is slightly less strict.

The last block gives the cost. Detecting a two-point difference with the corrected boundary needs 15351 users per group; 122808 users for eight combinations. Measuring the same difference with a single A/B test needs 19320 users. The multivariate test demands more than six times the traffic.

This determines when to choose a multivariate test. If an interaction is expected between the elements — if one’s effect depends on the other’s value — separate A/B tests cannot see that interaction at all, and a multivariate test is required. If no interaction is expected, separate, sequential A/B tests are both cheaper and easier to interpret.

Rules for the Experiment

The hypothesis and the sample size are written before the experiment. Which metric, in which direction, and by at least how much it must change is decided in advance. A metric chosen after the fact is the hidden form of the multiple-comparison problem from the fourth table.

No decision is made before the sample target is reached. The third computation gives the cost of not following this.

The difference is reported with its confidence interval. A point estimate alone hides the experiment’s uncertainty.

How many comparisons were made is written down. Results broken out by segment are comparisons too; a finding of “significant on narrow screens” means as many comparisons were made as there are segments.

The experiment does not change the legitimacy criterion. Showing that a change raises conversion does not show that the change passes the audit from the Dark Patterns and Ethics lesson. The experiment answers “does it work”; it does not answer “should it be done.”

Summary

• The required sample size is inversely proportional to the square of the detectable difference: 592 users per group for an 8-point difference, 38734 for a 1-point difference.
• The same measured difference (2.80 points) is inconclusive with groups of 2000 and significant with groups of 8000; a difference cannot be reported without the sample size.
• The width of the confidence interval is a separate piece of decision information; being significant does not mean being large enough to change the decision.
• Early stopping inflates the false-positive rate: with the true difference at zero, the test that looked only at the end said “difference found” 5.7 percent of the time, and the test that peeked every 500 users said so 20.4 percent of the time.
• In a multivariate test, the family-wise error rises with the number of comparisons (30.2 percent at 7 comparisons); the Bonferroni or Šidák correction tightens the boundary and raises the required traffic to more than six times in this example.
• The experiment shows whether a change works; whether it should be made is decided by a separate criterion.

Next Step

This lesson taught how to tell whether a difference is real; it did not say why the difference exists. If the change on the borrow confirmation screen raised the rate by 2.8 points, what users did differently does not show up in the number. The course’s last lesson closes this gap: the drop-off rate at the same funnel step is placed side by side with the codes from interviews conducted at that step, it is worked out which qualitative observation matches the quantitative finding, and what to do when the two sources contradict each other is addressed.