Calculate AB significance with Control and Treatment Objects

Question

I am working on an AB tool in JavaScript using the following:

1. Calculate the Conversion rate based on the conversions divided by the hits.

2. Calculate the Z Score by passing in the control and treatment objects.

3. Determine the confidence of the Z Score with Cumulative Normal Distribution.

How accurate is this?

/**
  Example input:

  var a = {
    'hits' : 15,
    'conversions' : 2
    };

  var b = {
     'hits' : 15,
     'conversions' : 6
  };
 */

/**
 * Calculation of conversion rate
 * @param t
 * @returns {number}
 */
function cr(t) {
    return t.conversions / t.hits;
}

/**
 * Calculation of score
 * @param c
 * @param t
 */
function calcZScore(c, t) {
    var z = cr(t) - cr(c);
    var s = (cr(t) * (1 - cr(t))) / t.hits + (cr(c) * (1 - cr(c))) / c.hits;
    return Math.sqrt(s);
}

/**
 * Calculate the Cumulative Normal Distribtion
 *
 * @param x
 * @returns {number}
 */
function cumNorDist(x) {
    var b1 = 0.319381530;
    var b2 = -0.356563782;
    var b3 = 1.781477937;
    var b4 = -1.821255978;
    var b5 = 1.330274429;
    var p = 0.2316419;
    var c = 0.39894228;

    if (x >= 0.0) {
        t = 1.0 / ( 1.0 + p * x );
        return (1.0 - c * Math.exp(-x * x / 2.0) * t *
        ( t * ( t * ( t * ( t * b5 + b4 ) + b3 ) + b2 ) + b1 ));
    } else {
        t = 1.0 / ( 1.0 - p * x );
        return ( c * Math.exp(-x * x / 2.0) * t *
        ( t * ( t * ( t * ( t * b5 + b4 ) + b3 ) + b2 ) + b1 ));
    }
}

/**
 * Given a conversion rate, calculate a recommended sample
 * size
 * E.g: 0.25 worst, 0.15, 0.05 best at a 95% confidence
 * @param conv
 * @returns {Array}
 */
function sampleSize(conv) {
    var a = 3.84145882689;
    var res = [];
    var bs = [0.0625, 0.0225, 0.0025];
    var len = bs.length;
    for (var i = 0; i < len; i++) {
        res.push(((1 - conv) * a / (bs[i] * conv)));
    }
    return res;
}

/**
 * Calculate the significance between Control and Treatment [A/B/C]
 *
 * @param controlObj
 * @param treatmentObj
 */
function calculateSig(controlObj, treatmentObj) {

    if (typeof controlObj !== 'object') {
        console.log(outputErrorFor('Control Object'));
    }

    if (typeof controlObj !== 'object') {
        console.log(outputErrorFor('Control Object'));
    }

    var zScore;
    var confidence;
    var confidencePercentage;
    var cRatio;

    var cNumHits = controlObj.hits;
    var cNumConver = controlObj.conversions;
    var tNumHits = treatmentObj.hits;
    var tNumConver = treatmentObj.conversions;

    var cConversionRate = (cNumConver / cNumHits) * 100;
    var tConversionRate = (tNumConver / tNumHits) * 100;

    cConversionRate += '%';
    tConversionRate += '%';

    zScore = calcZScore(controlObj, treatmentObj);
    confidence = cumNorDist(zScore);
    confidencePercentage = confidence * 100;

    console.log("Control Hits is:", cNumHits);
    console.log("Control Conversions is:", cNumConver);
    console.log("Treatment Hits is:", tNumHits);
    console.log("Treatment Conversions is:", tNumConver);

    console.log("Results are in:");

    console.log("Control Conversion Rate is:", cConversionRate);
    console.log("Treatment Conversion Rate is:", tConversionRate);

    console.log("Z Score is:", zScore);
    console.log("Confidence is:", confidence);
    console.log("Confident percentage is:", confidencePercentage);
}

/**
 *
 * @param name
 * @returns {string}
 */
var outputErrorFor = function (name) {
    return 'The passed in parameter must be an object and contain two properties: "hits" and "conversions" for: ' + name;
};

This is a ported version of this repo. (credit to the author)

Side note: With the Z Score, how can I find the P Value?

Answer 1

One problem here is in calcZScore, where it should be:

return z/Math.sqrt(s);

Otherwise, it looks like you successfully ported the code. More generally, there are some caveats associated with using the normal assumption for A/B tests. You want to make sure that you have a sufficient number of data points for the normality assumption to hold (often expressed as np > 5 and n(1-p)>5), which the sample size estimator doesn't always seem to satisfy.

You also should also be aware of other incorrect methods of A/B testing, such as repeated significance errors as expressed here: http://www.evanmiller.org/how-not-to-run-an-ab-test.html. In fact, I'd say getting working confidence calculation code is just scratching the surface of proper A/B testing.

Also, assuming you are doing a two-sided hypothesis test, the P-value is simply 1-confidence.

console.log("P-Value is:", 1-confidence);

I compared the results this code produces to the following online A/B calculator and it seemed to match up (although I haven't validated the sample size recommendation, so someone else should probably take a look at that). https://vwo.com/ab-split-test-significance-calculator/