dnd-character: chi-squared tests (#584)

Author: keiravillekode

exercises/practice/dnd-character/.docs/instructions.append.md

@@ -0,0 +1,23 @@
+# Instructions append
+
+## Randomness in Zig
+
+The Zig standard library avoids hidden global state, so rather than keeping a random number generator of your own, your functions receive a [`std.Random`][random] interface to draw values from.
+The tests pass in a generator seeded with [`std.testing.random_seed`][random-seed], and check that generators with the same seed produce the same characters.
+
+## Checking randomness
+
+The distribution of your ability scores is checked by [chi-squared tests][chi-squared-test] at a [significance level][p-value] of `p < 0.0001`:
+
+- the scores returned by `ability` are compared with the expected distribution;
+- each of a character's six abilities is compared with the expected distribution;
+- the pattern of odd and even scores across a character's six abilities is compared with what independent abilities would produce.
+
+A correct implementation has less than a 0.01% chance of failing each test.
+
+See the instructions above for a definition of what an ability score is.
+
+[random]: https://ziglang.org/documentation/0.16.0/std/#std.Random
+[random-seed]: https://ziglang.org/documentation/0.16.0/std/#std.testing.random_seed
+[chi-squared-test]: https://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test
+[p-value]: https://en.wikipedia.org/wiki/P-value

exercises/practice/dnd-character/.meta/config.json

@@ -2,6 +2,9 @@
   "authors": [
     "ee7"
   ],
+  "contributors": [
+    "keiravillekode"
+  ],
   "files": {
     "solution": [
       "dnd_character.zig"

exercises/practice/dnd-character/.meta/example.zig

@@ -1,22 +1,18 @@
 const std = @import("std");
 
-var prng = std.Random.DefaultPrng.init(42);
-const random = prng.random();
+pub fn modifier(score: u8) i8 {
+    return @divFloor(@as(i8, @intCast(score)) - 10, 2);
+}
 
-pub fn ability() u8 {
+pub fn ability(random: std.Random) u8 {
     var lowest: u8 = std.math.maxInt(u8);
     var result: u8 = 0;
     for (0..4) |_| {
         const roll = random.intRangeAtMost(u8, 1, 6);
         result += roll;
         lowest = @min(lowest, roll);
     }
-    result -= lowest;
-    return result;
-}
-
-pub fn modifier(score: u8) i8 {
-    return @divFloor(@as(i8, @intCast(score)) - 10, 2);
+    return result - lowest;
 }
 
 pub const Character = struct {
@@ -28,16 +24,16 @@ pub const Character = struct {
     charisma: u8,
     hitpoints: u8,
 
-    pub fn init() Character {
-        const constitution = ability();
+    pub fn init(random: std.Random) Character {
+        const constitution = ability(random);
         return .{
-            .strength = ability(),
-            .dexterity = ability(),
+            .strength = ability(random),
+            .dexterity = ability(random),
             .constitution = constitution,
-            .intelligence = ability(),
-            .wisdom = ability(),
-            .charisma = ability(),
-            .hitpoints = @as(u8, @intCast(10 + modifier(constitution))),
+            .intelligence = ability(random),
+            .wisdom = ability(random),
+            .charisma = ability(random),
+            .hitpoints = @intCast(10 + modifier(constitution)),
         };
     }
 };

exercises/practice/dnd-character/.meta/supplements.json

@@ -0,0 +1,28 @@
+{
+  "cases": [
+    {
+      "description": "random ability is distributed correctly",
+      "property": "abilityDistribution",
+      "input": {},
+      "expected": "chi-squared test against the 4d6-drop-lowest distribution passes at p < 0.0001"
+    },
+    {
+      "description": "each character ability is distributed correctly",
+      "property": "characterDistribution",
+      "input": {},
+      "expected": "chi-squared test of each ability against the 4d6-drop-lowest distribution passes at p < 0.0001"
+    },
+    {
+      "description": "character abilities are independent",
+      "property": "characterParity",
+      "input": {},
+      "expected": "chi-squared test of the odd/even pattern of the six abilities passes at p < 0.0001"
+    },
+    {
+      "description": "character depends only on the random number generator",
+      "property": "sameSeed",
+      "input": {},
+      "expected": "generators with the same seed produce the same characters"
+    }
+  ]
+}

exercises/practice/dnd-character/dnd_character.zig

@@ -1,22 +1,26 @@
-pub fn modifier(score: i8) i8 {
+const std = @import("std");
+
+pub fn modifier(score: u8) i8 {
     _ = score;
     @compileError("please implement the modifier function");
 }
 
-pub fn ability() i8 {
+pub fn ability(random: std.Random) u8 {
+    _ = random;
     @compileError("please implement the ability function");
 }
 
 pub const Character = struct {
-    strength: i8,
-    dexterity: i8,
-    constitution: i8,
-    intelligence: i8,
-    wisdom: i8,
-    charisma: i8,
-    hitpoints: i8,
+    strength: u8,
+    dexterity: u8,
+    constitution: u8,
+    intelligence: u8,
+    wisdom: u8,
+    charisma: u8,
+    hitpoints: u8,
 
-    pub fn init() Character {
+    pub fn init(random: std.Random) Character {
+        _ = random;
         @compileError("please implement the init method");
     }
 };

exercises/practice/dnd-character/test_dnd_character.zig

@@ -18,6 +18,54 @@ fn isValid(c: Character) bool {
         (c.hitpoints == 10 + dnd_character.modifier(c.constitution));
 }
 
+fn abilityScores(c: Character) [6]u8 {
+    return .{ c.strength, c.dexterity, c.constitution, c.intelligence, c.wisdom, c.charisma };
+}
+
+/// The number of times each score from 3 to 18 arises among the
+/// 6 * 6 * 6 * 6 = 1296 equally likely rolls of four dice.
+const score_weights = [16]u64{ 1, 4, 10, 21, 38, 62, 91, 122, 148, 167, 172, 160, 131, 94, 54, 21 };
+
+/// The number of times each pattern of odd (1) and even (0) scores arises
+/// among the 1296 ^ 6 equally likely ways of rolling a character's six abilities.
+const parity_weights = blk: {
+    var odd: u64 = 0;
+    var even: u64 = 0;
+    for (score_weights, 3..) |weight, score| {
+        if (score % 2 == 1) odd += weight else even += weight;
+    }
+    var weights: [64]u64 = undefined;
+    for (&weights, 0..) |*weight, pattern| {
+        const odd_count = @popCount(@as(u6, @intCast(pattern)));
+        weight.* = std.math.pow(u64, odd, odd_count) * std.math.pow(u64, even, 6 - odd_count);
+    }
+    break :blk weights;
+};
+
+/// Number of samples in each statistical test.
+const sample_size = 100 * 1296;
+
+/// Upper critical values of the chi-squared distribution at p = 0.0001,
+/// for 16 - 1 and 64 - 1 degrees of freedom.
+const critical_value_15: f64 = 44.2633;
+const critical_value_63: f64 = 113.505;
+
+/// Pearson's chi-squared statistic for the observed `counts`, when the
+/// expected counts are proportional to `weights`.
+fn chiSquared(counts: []const u64, weights: []const u64) f64 {
+    var total_count: u64 = 0;
+    for (counts) |count| total_count += count;
+    var total_weight: u64 = 0;
+    for (weights) |weight| total_weight += weight;
+    var statistic: f64 = 0;
+    for (counts, weights) |count, weight| {
+        const expected = @as(f64, @floatFromInt(total_count)) * @as(f64, @floatFromInt(weight)) / @as(f64, @floatFromInt(total_weight));
+        const difference = @as(f64, @floatFromInt(count)) - expected;
+        statistic += difference * difference / expected;
+    }
+    return statistic;
+}
+
 test "ability modifier for score 3 is -4" {
     const expected: i8 = -4;
     const actual = dnd_character.modifier(3);
@@ -115,15 +163,72 @@ test "ability modifier for score 18 is +4" {
 }
 
 test "random ability is within range" {
+    var prng = std.Random.DefaultPrng.init(testing.random_seed);
+    const random = prng.random();
     for (0..20) |_| {
-        const actual = dnd_character.ability();
+        const actual = dnd_character.ability(random);
         try testing.expect(isValidAbilityScore(actual));
     }
 }
 
+test "random ability is distributed correctly" {
+    var prng = std.Random.DefaultPrng.init(testing.random_seed);
+    const random = prng.random();
+    var counts: [16]u64 = @splat(0);
+    for (0..sample_size) |_| {
+        const score = dnd_character.ability(random);
+        try testing.expect(isValidAbilityScore(score));
+        counts[score - 3] += 1;
+    }
+    try testing.expect(chiSquared(&counts, &score_weights) < critical_value_15);
+}
+
 test "random character is valid" {
+    var prng = std.Random.DefaultPrng.init(testing.random_seed);
+    const random = prng.random();
     for (0..20) |_| {
-        const character = Character.init();
+        const character = Character.init(random);
         try testing.expect(isValid(character));
     }
 }
+
+test "each character ability is distributed correctly" {
+    var prng = std.Random.DefaultPrng.init(testing.random_seed);
+    const random = prng.random();
+    var counts: [6][16]u64 = @splat(@splat(0));
+    for (0..sample_size) |_| {
+        const character = Character.init(random);
+        for (abilityScores(character), &counts) |score, *ability_counts| {
+            try testing.expect(isValidAbilityScore(score));
+            ability_counts[score - 3] += 1;
+        }
+    }
+    for (counts) |ability_counts| {
+        try testing.expect(chiSquared(&ability_counts, &score_weights) < critical_value_15);
+    }
+}
+
+test "character abilities are independent" {
+    var prng = std.Random.DefaultPrng.init(testing.random_seed);
+    const random = prng.random();
+    var counts: [64]u64 = @splat(0);
+    for (0..sample_size) |_| {
+        const character = Character.init(random);
+        var pattern: usize = 0;
+        for (abilityScores(character)) |score| {
+            pattern = 2 * pattern + score % 2;
+        }
+        counts[pattern] += 1;
+    }
+    try testing.expect(chiSquared(&counts, &parity_weights) < critical_value_63);
+}
+
+test "character depends only on the random number generator" {
+    var prng = std.Random.DefaultPrng.init(testing.random_seed);
+    var other_prng = std.Random.DefaultPrng.init(testing.random_seed);
+    for (0..20) |_| {
+        const character = Character.init(prng.random());
+        const other_character = Character.init(other_prng.random());
+        try testing.expectEqual(character, other_character);
+    }
+}