Makemore

Mix.install([
  {:explorer, "~> 0.7.2"},
  {:kino, "~> 0.12.1"},
  {:axon, "~> 0.6.0"},
  {:tucan, "~> 0.3.0"},
  {:kino_vega_lite, "~> 0.1.11"}
])

Helpers

alias VegaLite, as: Vl

VegaLite

defmodule Makemore do
  def itos(chars) do
    chars
    |> Enum.with_index(fn el, i -> {i + 1, el} end)
    |> Map.new()
  end

  def stoi(chars) do
    chars
    |> Enum.with_index(fn el, i -> {el, i + 1} end)
    |> Map.new()
  end

  def multinomial(input, key, opts) do
    samples_opt = Keyword.fetch!(opts, :samples)
    replace_opt = Keyword.fetch!(opts, :replace)
    {samples, _} = Nx.Random.choice(key, input, samples: samples_opt, replace: replace_opt)
    input = input |> Nx.to_list()

    samples
    |> Nx.to_list()
    |> Enum.map(fn sample ->
      Enum.find_index(input, fn pp -> sample == pp end)
    end)
    |> Nx.tensor()
  end
end

{:module, Makemore, <<70, 79, 82, 49, 0, 0, 12, ...>>, {:multinomial, 3}}

Reading and Exploring the Dataset

dataset_filename_input = Kino.Input.textarea("Paste full file path:")

words =
  dataset_filename_input
  |> Kino.Input.read()
  |> File.read!()
  |> String.split()

["emma", "olivia", "ava", "isabella", "sophia", "charlotte", "mia", "amelia", "harper", "evelyn",
 "abigail", "emily", "elizabeth", "mila", "ella", "avery", "sofia", "camila", "aria", "scarlett",
 "victoria", "madison", "luna", "grace", "chloe", "penelope", "layla", "riley", "zoey", "nora",
 "lily", "eleanor", "hannah", "lillian", "addison", "aubrey", "ellie", "stella", "natalie", "zoe",
 "leah", "hazel", "violet", "aurora", "savannah", "audrey", "brooklyn", "bella", "claire", "skylar",
 ...]

words |> Enum.take(10)

["emma", "olivia", "ava", "isabella", "sophia", "charlotte", "mia", "amelia", "harper", "evelyn"]

min_length = words |> Enum.map(&String.length/1) |> Enum.min()

2

max_length = words |> Enum.map(&String.length/1) |> Enum.max()

15

Counting Bigrams

bigrams =
  words
  |> Enum.flat_map(fn word ->
    word
    |> String.graphemes()
    |> then(fn chars -> ["."] ++ chars ++ ["."] end)
    |> Enum.chunk_every(2, 1, :discard)
    |> Enum.map(fn [a, b] -> {a, b} end)
  end)

[
  {".", "e"},
  {"e", "m"},
  {"m", "m"},
  {"m", "a"},
  {"a", "."},
  {".", "o"},
  {"o", "l"},
  {"l", "i"},
  {"i", "v"},
  {"v", "i"},
  {"i", "a"},
  {"a", "."},
  {".", "a"},
  {"a", "v"},
  {"v", "a"},
  {"a", "."},
  {".", "i"},
  {"i", "s"},
  {"s", "a"},
  {"a", "b"},
  {"b", "e"},
  {"e", "l"},
  {"l", "l"},
  {"l", "a"},
  {"a", "."},
  {".", "s"},
  {"s", "o"},
  {"o", "p"},
  {"p", "h"},
  {"h", "i"},
  {"i", "a"},
  {"a", "."},
  {".", "c"},
  {"c", "h"},
  {"h", "a"},
  {"a", "r"},
  {"r", "l"},
  {"l", "o"},
  {"o", "t"},
  {"t", "t"},
  {"t", "e"},
  {"e", "."},
  {".", "m"},
  {"m", "i"},
  {"i", "a"},
  {"a", "."},
  {".", "a"},
  {"a", "m"},
  {"m", ...},
  {...},
  ...
]

bigram_counts =
  bigrams
  |> Enum.reduce(Map.new(), fn bigram, acc ->
    Map.update(acc, bigram, 1, &(&1 + 1))
  end)
  |> Enum.sort_by(fn {_k, v} -> v end, :desc)

[
  {{"n", "."}, 6763},
  {{"a", "."}, 6640},
  {{"a", "n"}, 5438},
  {{".", "a"}, 4410},
  {{"e", "."}, 3983},
  {{"a", "r"}, 3264},
  {{"e", "l"}, 3248},
  {{"r", "i"}, 3033},
  {{"n", "a"}, 2977},
  {{".", "k"}, 2963},
  {{"l", "e"}, 2921},
  {{"e", "n"}, 2675},
  {{"l", "a"}, 2623},
  {{"m", "a"}, 2590},
  {{".", "m"}, 2538},
  {{"a", "l"}, 2528},
  {{"i", "."}, 2489},
  {{"l", "i"}, 2480},
  {{"i", "a"}, 2445},
  {{".", "j"}, 2422},
  {{"o", "n"}, 2411},
  {{"h", "."}, 2409},
  {{"r", "a"}, 2356},
  {{"a", "h"}, 2332},
  {{"h", "a"}, 2244},
  {{"y", "a"}, 2143},
  {{"i", "n"}, 2126},
  {{".", "s"}, 2055},
  {{"a", "y"}, 2050},
  {{"y", "."}, 2007},
  {{"e", "r"}, 1958},
  {{"n", "n"}, 1906},
  {{"y", "n"}, 1826},
  {{"k", "a"}, 1731},
  {{"n", "i"}, 1725},
  {{"r", "e"}, 1697},
  {{".", "d"}, 1690},
  {{"i", "e"}, 1653},
  {{"a", "i"}, 1650},
  {{".", "r"}, 1639},
  {{"a", "m"}, 1634},
  {{"l", "y"}, 1588},
  {{".", "l"}, 1572},
  {{".", "c"}, 1542},
  {{".", "e"}, 1531},
  {{"j", "a"}, 1473},
  {{"r", "."}, 1377},
  {{"n", ...}, 1359},
  {{...}, ...},
  {...},
  ...
]

Counting Bigrams in 2D Tensor - “Training The Model”

chars =
  words
  |> Enum.flat_map(&String.graphemes/1)
  |> Enum.uniq()
  |> Enum.sort()

["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t",
 "u", "v", "w", "x", "y", "z"]

stoi =
  chars
  |> Makemore.stoi()
  |> Map.put(".", 0)

%{
  "." => 0,
  "a" => 1,
  "b" => 2,
  "c" => 3,
  "d" => 4,
  "e" => 5,
  "f" => 6,
  "g" => 7,
  "h" => 8,
  "i" => 9,
  "j" => 10,
  "k" => 11,
  "l" => 12,
  "m" => 13,
  "n" => 14,
  "o" => 15,
  "p" => 16,
  "q" => 17,
  "r" => 18,
  "s" => 19,
  "t" => 20,
  "u" => 21,
  "v" => 22,
  "w" => 23,
  "x" => 24,
  "y" => 25,
  "z" => 26
}

itos =
  chars
  |> Makemore.itos()
  |> Map.put(0, ".")

%{
  0 => ".",
  1 => "a",
  2 => "b",
  3 => "c",
  4 => "d",
  5 => "e",
  6 => "f",
  7 => "g",
  8 => "h",
  9 => "i",
  10 => "j",
  11 => "k",
  12 => "l",
  13 => "m",
  14 => "n",
  15 => "o",
  16 => "p",
  17 => "q",
  18 => "r",
  19 => "s",
  20 => "t",
  21 => "u",
  22 => "v",
  23 => "w",
  24 => "x",
  25 => "y",
  26 => "z"
}

bigram_count_map =
  bigrams
  |> Enum.reduce(Map.new(), fn {ch1, ch2}, acc ->
    ix1 = Map.fetch!(stoi, ch1)
    ix2 = Map.fetch!(stoi, ch2)
    Map.update(acc, {ix1, ix2}, 1, fn ch_count -> ch_count + 1 end)
  end)

%{
  {18, 15} => 869,
  {9, 24} => 89,
  {18, 21} => 252,
  {12, 12} => 1345,
  {5, 23} => 50,
  {22, 1} => 642,
  {21, 5} => 169,
  {0, 20} => 1308,
  {24, 3} => 4,
  {12, 1} => 2623,
  {11, 13} => 9,
  {0, 10} => 2422,
  {23, 7} => 1,
  {14, 21} => 96,
  {17, 21} => 206,
  {11, 3} => 2,
  {3, 26} => 4,
  {9, 20} => 541,
  {15, 16} => 95,
  {24, 5} => 36,
  {9, 8} => 95,
  {26, 24} => 1,
  {23, 26} => 1,
  {24, 8} => 1,
  {15, 17} => 3,
  {14, 9} => 1725,
  {13, 18} => 97,
  {23, 11} => 6,
  {13, 4} => 24,
  {23, 9} => 148,
  {10, 14} => 2,
  {4, 15} => 378,
  {5, 10} => 55,
  {23, 1} => 280,
  {15, 23} => 114,
  {9, 4} => 440,
  {14, 15} => 496,
  {20, 8} => 647,
  {21, 9} => 121,
  {9, 25} => 779,
  {21, 19} => 474,
  {1, 7} => 168,
  {25, 11} => 86,
  {2, 20} => 2,
  {9, 19} => 1316,
  {7, 14} => 27,
  {0, 13} => 2538,
  {21, 23} => 86,
  {18, ...} => 121,
  {...} => 41,
  ...
}

bigram_count_list =
  for x <- 0..27 do
    for y <- 0..27 do
      Map.get(bigram_count_map, {x, y}, 0)
    end
  end

[
  [0, 4410, 1306, 1542, 1690, 1531, 417, 669, 874, 591, 2422, 2963, 1572, 2538, 1146, 394, 515, 92,
   1639, 2055, 1308, 78, 376, 307, 134, 535, 929, 0],
  [6640, 556, 541, 470, 1042, 692, 134, 168, 2332, 1650, 175, 568, 2528, 1634, 5438, 63, 82, 60,
   3264, 1118, 687, 381, 834, 161, 182, 2050, 435, 0],
  [114, 321, 38, 1, 65, 655, 0, 0, 41, 217, 1, 0, 103, 0, 4, 105, 0, 0, 842, 8, 2, 45, 0, 0, 0, 83,
   0, 0],
  [97, 815, 0, 42, 1, 551, 0, 2, 664, 271, 3, 316, 116, 0, 0, 380, 1, 11, 76, 5, 35, 35, 0, 0, 3,
   104, 4, 0],
  [516, 1303, 1, 3, 149, 1283, 5, 25, 118, 674, 9, 3, 60, 30, 31, 378, 0, 1, 424, 29, 4, 92, 17, 23,
   0, 317, 1, 0],
  [3983, 679, 121, 153, 384, 1271, 82, 125, 152, 818, 55, 178, 3248, 769, 2675, 269, 83, 14, 1958,
   861, 580, 69, 463, 50, 132, 1070, 181, 0],
  [80, 242, 0, 0, 0, 123, 44, 1, 1, 160, 0, 2, 20, 0, 4, 60, 0, 0, 114, 6, 18, 10, 0, 4, 0, 14, 2,
   0],
  [108, 330, 3, 0, 19, 334, 1, 25, 360, 190, 3, 0, 32, 6, 27, 83, 0, 0, 201, 30, 31, 85, 1, 26, 0,
   31, 1, 0],
  [2409, 2244, 8, 2, 24, 674, 2, 2, 1, 729, 9, 29, 185, 117, 138, 287, 1, 1, 204, 31, 71, 166, 39,
   10, 0, 213, 20, 0],
  [2489, 2445, 110, 509, 440, 1653, 101, 428, 95, 82, 76, 445, 1345, 427, 2126, 588, 53, 52, 849,
   1316, 541, 109, 269, 8, 89, 779, 277, 0],
  [71, 1473, 1, 4, 4, 440, 0, 0, 45, 119, 2, 2, 9, 5, 2, 479, 1, 0, 11, 7, 2, 202, 5, 6, 0, 10, 0,
   0],
  [363, 1731, 2, 2, 2, 895, 1, 0, 307, 509, 2, 20, 139, 9, 26, 344, 0, 0, 109, 95, 17, 50, 2, 34, 0,
   379, 2, 0],
  [1314, 2623, 52, 25, 138, 2921, 22, 6, 19, 2480, 6, 24, 1345, 60, 14, 692, 15, 3, 18, 94, 77, 324,
   72, 16, 0, 1588, 10, 0],
  [516, 2590, 112, 51, 24, 818, 1, 0, 5, 1256, 7, 1, 5, 168, 20, 452, 38, 0, 97, 35, 4, 139, 3, 2,
   0, 287, 11, 0],
  [6763, 2977, 8, 213, 704, 1359, 11, 273, 26, 1725, 44, 58, 195, 19, 1906, 496, 5, 2, 44, 278, 443,
   96, 55, 11, 6, 465, 145, 0],
  [855, 149, 140, 114, 190, 132, 34, 44, 171, 69, 16, 68, 619, 261, 2411, 115, 95, 3, 1059, 504,
   118, 275, 176, 114, 45, 103, 54, 0],
  [33, 209, 2, 1, 0, 197, 1, 0, 204, 61, 1, 1, 16, 1, 1, 59, 39, 0, 151, 16, 17, 4, 0, 0, 0, 12, 0,
   0],
  [28, 13, 0, 0, 0, 1, 0, 0, 0, 13, 0, 0, 1, 2, 0, 2, 0, 0, 1, 2, 0, 206, 0, 3, 0, 0, 0, 0],
  [1377, 2356, 41, 99, 187, 1697, 9, 76, 121, 3033, 25, 90, 413, 162, 140, 869, 14, 16, 425, 190,
   208, 252, 80, 21, 3, 773, 23, 0],
  [1169, 1201, 21, 60, 9, 884, 2, 2, 1285, 684, 2, 82, 279, 90, 24, 531, 51, 1, 55, 461, 765, 185,
   14, 24, 0, 215, 10, 0],
  [483, 1027, 1, 17, 0, 716, 2, 2, 647, 532, 3, 0, 134, 4, 22, 667, 0, 0, 352, 35, 374, 78, 15, 11,
   2, 341, 105, 0],
  [155, 163, 103, 103, 136, 169, 19, 47, 58, 121, 14, 93, 301, 154, 275, 10, 16, 10, 414, 474, 82,
   3, 37, 86, 34, 13, 45, 0],
  [88, 642, 1, 0, 1, 568, 0, 0, 1, 911, 0, 3, 14, 0, 8, 153, 0, 0, 48, 0, 0, 7, 7, 0, 0, 121, 0,
   ...],
  [51, 280, 1, 0, 8, 149, 2, 1, 23, 148, 0, 6, 13, 2, 58, 36, 0, 0, 22, 20, 8, 25, 0, 2, 0, 73, ...],
  [164, 103, 1, 4, 5, 36, 3, 0, 1, 102, 0, 0, 39, 1, 1, 41, 0, 0, 0, 31, 70, 5, 0, 3, 38, ...],
  [2007, 2143, 27, 115, 272, 301, 12, 30, 22, 192, 23, 86, 1104, 148, 1826, 271, 15, 6, 291, 401,
   104, 141, 106, 4, ...],
  [160, 860, 4, 2, 2, 373, 0, 1, 43, 364, 2, 2, 123, 35, 4, 110, 2, 0, 32, 4, 4, 73, 2, ...],
  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...]
]

bigram_count_tensor = bigram_count_list |> Nx.tensor(type: {:f, 32})

#Nx.Tensor<
  f32[28][28]
  [
    [0.0, 4410.0, 1306.0, 1542.0, 1690.0, 1531.0, 417.0, 669.0, 874.0, 591.0, 2422.0, 2963.0, 1572.0, 2538.0, 1146.0, 394.0, 515.0, 92.0, 1639.0, 2055.0, 1308.0, 78.0, 376.0, 307.0, 134.0, 535.0, 929.0, 0.0],
    [6640.0, 556.0, 541.0, 470.0, 1042.0, 692.0, 134.0, 168.0, 2332.0, 1650.0, 175.0, 568.0, 2528.0, 1634.0, 5438.0, 63.0, 82.0, 60.0, 3264.0, 1118.0, 687.0, 381.0, ...],
    ...
  ]
>

Visualization of the Bigram Counts

bigram_count_tensor
|> Tucan.imshow(width: 700, height: 700, show_scale: true, tooltip: true, color_scheme: :blues)
|> Tucan.set_title("Bigram Count")

{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"v":0.0,"x":0,"y":0},{"v":4410.0,"x":1,"y":0},{"v":1306.0,"x":2,"y":0},{"v":1542.0,"x":3,"y":0},{"v":1690.0,"x":4,"y":0},{"v":1531.0,"x":5,"y":0},{"v":417.0,"x":6,"y":0},{"v":669.0,"x":7,"y":0},{"v":874.0,"x":8,"y":0},{"v":591.0,"x":9,"y":0},{"v":2422.0,"x":10,"y":0},{"v":2963.0,"x":11,"y":0},{"v":1572.0,"x":12,"y":0},{"v":2538.0,"x":13,"y":0},{"v":1146.0,"x":14,"y":0},{"v":394.0,"x":15,"y":0},{"v":515.0,"x":16,"y":0},{"v":92.0,"x":17,"y":0},{"v":1639.0,"x":18,"y":0},{"v":2055.0,"x":19,"y":0},{"v":1308.0,"x":20,"y":0},{"v":78.0,"x":21,"y":0},{"v":376.0,"x":22,"y":0},{"v":307.0,"x":23,"y":0},{"v":134.0,"x":24,"y":0},{"v":535.0,"x":25,"y":0},{"v":929.0,"x":26,"y":0},{"v":0.0,"x":27,"y":0},{"v":6640.0,"x":0,"y":1},{"v":556.0,"x":1,"y":1},{"v":541.0,"x":2,"y":1},{"v":470.0,"x":3,"y":1},{"v":1042.0,"x":4,"y":1},{"v":692.0,"x":5,"y":1},{"v":134.0,"x":6,"y":1},{"v":168.0,"x":7,"y":1},{"v":2332.0,"x":8,"y":1},{"v":1650.0,"x":9,"y":1},{"v":175.0,"x":10,"y":1},{"v":568.0,"x":11,"y":1},{"v":2528.0,"x":12,"y":1},{"v":1634.0,"x":13,"y":1},{"v":5438.0,"x":14,"y":1},{"v":63.0,"x":15,"y":1},{"v":82.0,"x":16,"y":1},{"v":60.0,"x":17,"y":1},{"v":3264.0,"x":18,"y":1},{"v":1118.0,"x":19,"y":1},{"v":687.0,"x":20,"y":1},{"v":381.0,"x":21,"y":1},{"v":834.0,"x":22,"y":1},{"v":161.0,"x":23,"y":1},{"v":182.0,"x":24,"y":1},{"v":2050.0,"x":25,"y":1},{"v":435.0,"x":26,"y":1},{"v":0.0,"x":27,"y":1},{"v":114.0,"x":0,"y":2},{"v":321.0,"x":1,"y":2},{"v":38.0,"x":2,"y":2},{"v":1.0,"x":3,"y":2},{"v":65.0,"x":4,"y":2},{"v":655.0,"x":5,"y":2},{"v":0.0,"x":6,"y":2},{"v":0.0,"x":7,"y":2},{"v":41.0,"x":8,"y":2},{"v":217.0,"x":9,"y":2},{"v":1.0,"x":10,"y":2},{"v":0.0,"x":11,"y":2},{"v":103.0,"x":12,"y":2},{"v":0.0,"x":13,"y":2},{"v":4.0,"x":14,"y":2},{"v":105.0,"x":15,"y":2},{"v":0.0,"x":16,"y":2},{"v":0.0,"x":17,"y":2},{"v":842.0,"x":18,"y":2},{"v":8.0,"x":19,"y":2},{"v":2.0,"x":20,"y":2},{"v":45.0,"x":21,"y":2},{"v":0.0,"x":22,"y":2},{"v":0.0,"x":23,"y":2},{"v":0.0,"x":24,"y":2},{"v":83.0,"x":25,"y":2},{"v":0.0,"x":26,"y":2},{"v":0.0,"x":27,"y":2},{"v":97.0,"x":0,"y":3},{"v":815.0,"x":1,"y":3},{"v":0.0,"x":2,"y":3},{"v":42.0,"x":3,"y":3},{"v":1.0,"x":4,"y":3},{"v":551.0,"x":5,"y":3},{"v":0.0,"x":6,"y":3},{"v":2.0,"x":7,"y":3},{"v":664.0,"x":8,"y":3},{"v":271.0,"x":9,"y":3},{"v":3.0,"x":10,"y":3},{"v":316.0,"x":11,"y":3},{"v":116.0,"x":12,"y":3},{"v":0.0,"x":13,"y":3},{"v":0.0,"x":14,"y":3},{"v":380.0,"x":15,"y":3},{"v":1.0,"x":16,"y":3},{"v":11.0,"x":17,"y":3},{"v":76.0,"x":18,"y":3},{"v":5.0,"x":19,"y":3},{"v":35.0,"x":20,"y":3},{"v":35.0,"x":21,"y":3},{"v":0.0,"x":22,"y":3},{"v":0.0,"x":23,"y":3},{"v":3.0,"x":24,"y":3},{"v":104.0,"x":25,"y":3},{"v":4.0,"x":26,"y":3},{"v":0.0,"x":27,"y":3},{"v":516.0,"x":0,"y":4},{"v":1303.0,"x":1,"y":4},{"v":1.0,"x":2,"y":4},{"v":3.0,"x":3,"y":4},{"v":149.0,"x":4,"y":4},{"v":1283.0,"x":5,"y":4},{"v":5.0,"x":6,"y":4},{"v":25.0,"x":7,"y":4},{"v":118.0,"x":8,"y":4},{"v":674.0,"x":9,"y":4},{"v":9.0,"x":10,"y":4},{"v":3.0,"x":11,"y":4},{"v":60.0,"x":12,"y":4},{"v":30.0,"x":13,"y":4},{"v":31.0,"x":14,"y":4},{"v":378.0,"x":15,"y":4},{"v":0.0,"x":16,"y":4},{"v":1.0,"x":17,"y":4},{"v":424.0,"x":18,"y":4},{"v":29.0,"x":19,"y":4},{"v":4.0,"x":20,"y":4},{"v":92.0,"x":21,"y":4},{"v":17.0,"x":22,"y":4},{"v":23.0,"x":23,"y":4},{"v":0.0,"x":24,"y":4},{"v":317.0,"x":25,"y":4},{"v":1.0,"x":26,"y":4},{"v":0.0,"x":27,"y":4},{"v":3983.0,"x":0,"y":5},{"v":679.0,"x":1,"y":5},{"v":121.0,"x":2,"y":5},{"v":153.0,"x":3,"y":5},{"v":384.0,"x":4,"y":5},{"v":1271.0,"x":5,"y":5},{"v":82.0,"x":6,"y":5},{"v":125.0,"x":7,"y":5},{"v":152.0,"x":8,"y":5},{"v":818.0,"x":9,"y":5},{"v":55.0,"x":10,"y":5},{"v":178.0,"x":11,"y":5},{"v":3248.0,"x":12,"y":5},{"v":769.0,"x":13,"y":5},{"v":2675.0,"x":14,"y":5},{"v":269.0,"x":15,"y":5},{"v":83.0,"x":16,"y":5},{"v":14.0,"x":17,"y":5},{"v":1958.0,"x":18,"y":5},{"v":861.0,"x":19,"y":5},{"v":580.0,"x":20,"y":5},{"v":69.0,"x":21,"y":5},{"v":463.0,"x":22,"y":5},{"v":50.0,"x":23,"y":5},{"v":132.0,"x":24,"y":5},{"v":1070.0,"x":25,"y":5},{"v":181.0,"x":26,"y":5},{"v":0.0,"x":27,"y":5},{"v":80.0,"x":0,"y":6},{"v":242.0,"x":1,"y":6},{"v":0.0,"x":2,"y":6},{"v":0.0,"x":3,"y":6},{"v":0.0,"x":4,"y":6},{"v":123.0,"x":5,"y":6},{"v":44.0,"x":6,"y":6},{"v":1.0,"x":7,"y":6},{"v":1.0,"x":8,"y":6},{"v":160.0,"x":9,"y":6},{"v":0.0,"x":10,"y":6},{"v":2.0,"x":11,"y":6},{"v":20.0,"x":12,"y":6},{"v":0.0,"x":13,"y":6},{"v":4.0,"x":14,"y":6},{"v":60.0,"x":15,"y":6},{"v":0.0,"x":16,"y":6},{"v":0.0,"x":17,"y":6},{"v":114.0,"x":18,"y":6},{"v":6.0,"x":19,"y":6},{"v":18.0,"x":20,"y":6},{"v":10.0,"x":21,"y":6},{"v":0.0,"x":22,"y":6},{"v":4.0,"x":23,"y":6},{"v":0.0,"x":24,"y":6},{"v":14.0,"x":25,"y":6},{"v":2.0,"x":26,"y":6},{"v":0.0,"x":27,"y":6},{"v":108.0,"x":0,"y":7},{"v":330.0,"x":1,"y":7},{"v":3.0,"x":2,"y":7},{"v":0.0,"x":3,"y":7},{"v":19.0,"x":4,"y":7},{"v":334.0,"x":5,"y":7},{"v":1.0,"x":6,"y":7},{"v":25.0,"x":7,"y":7},{"v":360.0,"x":8,"y":7},{"v":190.0,"x":9,"y":7},{"v":3.0,"x":10,"y":7},{"v":0.0,"x":11,"y":7},{"v":32.0,"x":12,"y":7},{"v":6.0,"x":13,"y":7},{"v":27.0,"x":14,"y":7},{"v":83.0,"x":15,"y":7},{"v":0.0,"x":16,"y":7},{"v":0.0,"x":17,"y":7},{"v":201.0,"x":18,"y":7},{"v":30.0,"x":19,"y":7},{"v":31.0,"x":20,"y":7},{"v":85.0,"x":21,"y":7},{"v":1.0,"x":22,"y":7},{"v":26.0,"x":23,"y":7},{"v":0.0,"x":24,"y":7},{"v":31.0,"x":25,"y":7},{"v":1.0,"x":26,"y":7},{"v":0.0,"x":27,"y":7},{"v":2409.0,"x":0,"y":8},{"v":2244.0,"x":1,"y":8},{"v":8.0,"x":2,"y":8},{"v":2.0,"x":3,"y":8},{"v":24.0,"x":4,"y":8},{"v":674.0,"x":5,"y":8},{"v":2.0,"x":6,"y":8},{"v":2.0,"x":7,"y":8},{"v":1.0,"x":8,"y":8},{"v":729.0,"x":9,"y":8},{"v":9.0,"x":10,"y":8},{"v":29.0,"x":11,"y":8},{"v":185.0,"x":12,"y":8},{"v":117.0,"x":13,"y":8},{"v":138.0,"x":14,"y":8},{"v":287.0,"x":15,"y":8},{"v":1.0,"x":16,"y":8},{"v":1.0,"x":17,"y":8},{"v":204.0,"x":18,"y":8},{"v":31.0,"x":19,"y":8},{"v":71.0,"x":20,"y":8},{"v":166.0,"x":21,"y":8},{"v":39.0,"x":22,"y":8},{"v":10.0,"x":23,"y":8},{"v":0.0,"x":24,"y":8},{"v":213.0,"x":25,"y":8},{"v":20.0,"x":26,"y":8},{"v":0.0,"x":27,"y":8},{"v":2489.0,"x":0,"y":9},{"v":2445.0,"x":1,"y":9},{"v":110.0,"x":2,"y":9},{"v":509.0,"x":3,"y":9},{"v":440.0,"x":4,"y":9},{"v":1653.0,"x":5,"y":9},{"v":101.0,"x":6,"y":9},{"v":428.0,"x":7,"y":9},{"v":95.0,"x":8,"y":9},{"v":82.0,"x":9,"y":9},{"v":76.0,"x":10,"y":9},{"v":445.0,"x":11,"y":9},{"v":1345.0,"x":12,"y":9},{"v":427.0,"x":13,"y":9},{"v":2126.0,"x":14,"y":9},{"v":588.0,"x":15,"y":9},{"v":53.0,"x":16,"y":9},{"v":52.0,"x":17,"y":9},{"v":849.0,"x":18,"y":9},{"v":1316.0,"x":19,"y":9},{"v":541.0,"x":20,"y":9},{"v":109.0,"x":21,"y":9},{"v":269.0,"x":22,"y":9},{"v":8.0,"x":23,"y":9},{"v":89.0,"x":24,"y":9},{"v":779.0,"x":25,"y":9},{"v":277.0,"x":26,"y":9},{"v":0.0,"x":27,"y":9},{"v":71.0,"x":0,"y":10},{"v":1473.0,"x":1,"y":10},{"v":1.0,"x":2,"y":10},{"v":4.0,"x":3,"y":10},{"v":4.0,"x":4,"y":10},{"v":440.0,"x":5,"y":10},{"v":0.0,"x":6,"y":10},{"v":0.0,"x":7,"y":10},{"v":45.0,"x":8,"y":10},{"v":119.0,"x":9,"y":10},{"v":2.0,"x":10,"y":10},{"v":2.0,"x":11,"y":10},{"v":9.0,"x":12,"y":10},{"v":5.0,"x":13,"y":10},{"v":2.0,"x":14,"y":10},{"v":479.0,"x":15,"y":10},{"v":1.0,"x":16,"y":10},{"v":0.0,"x":17,"y":10},{"v":11.0,"x":18,"y":10},{"v":7.0,"x":19,"y":10},{"v":2.0,"x":20,"y":10},{"v":202.0,"x":21,"y":10},{"v":5.0,"x":22,"y":10},{"v":6.0,"x":23,"y":10},{"v":0.0,"x":24,"y":10},{"v":10.0,"x":25,"y":10},{"v":0.0,"x":26,"y":10},{"v":0.0,"x":27,"y":10},{"v":363.0,"x":0,"y":11},{"v":1731.0,"x":1,"y":11},{"v":2.0,"x":2,"y":11},{"v":2.0,"x":3,"y":11},{"v":2.0,"x":4,"y":11},{"v":895.0,"x":5,"y":11},{"v":1.0,"x":6,"y":11},{"v":0.0,"x":7,"y":11},{"v":307.0,"x":8,"y":11},{"v":509.0,"x":9,"y":11},{"v":2.0,"x":10,"y":11},{"v":20.0,"x":11,"y":11},{"v":139.0,"x":12,"y":11},{"v":9.0,"x":13,"y":11},{"v":26.0,"x":14,"y":11},{"v":344.0,"x":15,"y":11},{"v":0.0,"x":16,"y":11},{"v":0.0,"x":17,"y":11},{"v":109.0,"x":18,"y":11},{"v":95.0,"x":19,"y":11},{"v":17.0,"x":20,"y":11},{"v":50.0,"x":21,"y":11},{"v":2.0,"x":22,"y":11},{"v":34.0,"x":23,"y":11},{"v":0.0,"x":24,"y":11},{"v":379.0,"x":25,"y":11},{"v":2.0,"x":26,"y":11},{"v":0.0,"x":27,"y":11},{"v":1314.0,"x":0,"y":12},{"v":2623.0,"x":1,"y":12},{"v":52.0,"x":2,"y":12},{"v":25.0,"x":3,"y":12},{"v":138.0,"x":4,"y":12},{"v":2921.0,"x":5,"y":12},{"v":22.0,"x":6,"y":12},{"v":6.0,"x":7,"y":12},{"v":19.0,"x":8,"y":12},{"v":2480.0,"x":9,"y":12},{"v":6.0,"x":10,"y":12},{"v":24.0,"x":11,"y":12},{"v":1345.0,"x":12,"y":12},{"v":60.0,"x":13,"y":12},{"v":14.0,"x":14,"y":12},{"v":692.0,"x":15,"y":12},{"v":15.0,"x":16,"y":12},{"v":3.0,"x":17,"y":12},{"v":18.0,"x":18,"y":12},{"v":94.0,"x":19,"y":12},{"v":77.0,"x":20,"y":12},{"v":324.0,"x":21,"y":12},{"v":72.0,"x":22,"y":12},{"v":16.0,"x":23,"y":12},{"v":0.0,"x":24,"y":12},{"v":1588.0,"x":25,"y":12},{"v":10.0,"x":26,"y":12},{"v":0.0,"x":27,"y":12},{"v":516.0,"x":0,"y":13},{"v":2590.0,"x":1,"y":13},{"v":112.0,"x":2,"y":13},{"v":51.0,"x":3,"y":13},{"v":24.0,"x":4,"y":13},{"v":818.0,"x":5,"y":13},{"v":1.0,"x":6,"y":13},{"v":0.0,"x":7,"y":13},{"v":5.0,"x":8,"y":13},{"v":1256.0,"x":9,"y":13},{"v":7.0,"x":10,"y":13},{"v":1.0,"x":11,"y":13},{"v":5.0,"x":12,"y":13},{"v":168.0,"x":13,"y":13},{"v":20.0,"x":14,"y":13},{"v":452.0,"x":15,"y":13},{"v":38.0,"x":16,"y":13},{"v":0.0,"x":17,"y":13},{"v":97.0,"x":18,"y":13},{"v":35.0,"x":19,"y":13},{"v":4.0,"x":20,"y":13},{"v":139.0,"x":21,"y":13},{"v":3.0,"x":22,"y":13},{"v":2.0,"x":23,"y":13},{"v":0.0,"x":24,"y":13},{"v":287.0,"x":25,"y":13},{"v":11.0,"x":26,"y":13},{"v":0.0,"x":27,"y":13},{"v":6763.0,"x":0,"y":14},{"v":2977.0,"x":1,"y":14},{"v":8.0,"x":2,"y":14},{"v":213.0,"x":3,"y":14},{"v":704.0,"x":4,"y":14},{"v":1359.0,"x":5,"y":14},{"v":11.0,"x":6,"y":14},{"v":273.0,"x":7,"y":14},{"v":26.0,"x":8,"y":14},{"v":1725.0,"x":9,"y":14},{"v":44.0,"x":10,"y":14},{"v":58.0,"x":11,"y":14},{"v":195.0,"x":12,"y":14},{"v":19.0,"x":13,"y":14},{"v":1906.0,"x":14,"y":14},{"v":496.0,"x":15,"y":14},{"v":5.0,"x":16,"y":14},{"v":2.0,"x":17,"y":14},{"v":44.0,"x":18,"y":14},{"v":278.0,"x":19,"y":14},{"v":443.0,"x":20,"y":14},{"v":96.0,"x":21,"y":14},{"v":55.0,"x":22,"y":14},{"v":11.0,"x":23,"y":14},{"v":6.0,"x":24,"y":14},{"v":465.0,"x":25,"y":14},{"v":145.0,"x":26,"y":14},{"v":0.0,"x":27,"y":14},{"v":855.0,"x":0,"y":15},{"v":149.0,"x":1,"y":15},{"v":140.0,"x":2,"y":15},{"v":114.0,"x":3,"y":15},{"v":190.0,"x":4,"y":15},{"v":132.0,"x":5,"y":15},{"v":34.0,"x":6,"y":15},{"v":44.0,"x":7,"y":15},{"v":171.0,"x":8,"y":15},{"v":69.0,"x":9,"y":15},{"v":16.0,"x":10,"y":15},{"v":68.0,"x":11,"y":15},{"v":619.0,"x":12,"y":15},{"v":261.0,"x":13,"y":15},{"v":2411.0,"x":14,"y":15},{"v":115.0,"x":15,"y":15},{"v":95.0,"x":16,"y":15},{"v":3.0,"x":17,"y":15},{"v":1059.0,"x":18,"y":15},{"v":504.0,"x":19,"y":15},{"v":118.0,"x":20,"y":15},{"v":275.0,"x":21,"y":15},{"v":176.0,"x":22,"y":15},{"v":114.0,"x":23,"y":15},{"v":45.0,"x":24,"y":15},{"v":103.0,"x":25,"y":15},{"v":54.0,"x":26,"y":15},{"v":0.0,"x":27,"y":15},{"v":33.0,"x":0,"y":16},{"v":209.0,"x":1,"y":16},{"v":2.0,"x":2,"y":16},{"v":1.0,"x":3,"y":16},{"v":0.0,"x":4,"y":16},{"v":197.0,"x":5,"y":16},{"v":1.0,"x":6,"y":16},{"v":0.0,"x":7,"y":16},{"v":204.0,"x":8,"y":16},{"v":61.0,"x":9,"y":16},{"v":1.0,"x":10,"y":16},{"v":1.0,"x":11,"y":16},{"v":16.0,"x":12,"y":16},{"v":1.0,"x":13,"y":16},{"v":1.0,"x":14,"y":16},{"v":59.0,"x":15,"y":16},{"v":39.0,"x":16,"y":16},{"v":0.0,"x":17,"y":16},{"v":151.0,"x":18,"y":16},{"v":16.0,"x":19,"y":16},{"v":17.0,"x":20,"y":16},{"v":4.0,"x":21,"y":16},{"v":0.0,"x":22,"y":16},{"v":0.0,"x":23,"y":16},{"v":0.0,"x":24,"y":16},{"v":12.0,"x":25,"y":16},{"v":0.0,"x":26,"y":16},{"v":0.0,"x":27,"y":16},{"v":28.0,"x":0,"y":17},{"v":13.0,"x":1,"y":17},{"v":0.0,"x":2,"y":17},{"v":0.0,"x":3,"y":17},{"v":0.0,"x":4,"y":17},{"v":1.0,"x":5,"y":17},{"v":0.0,"x":6,"y":17},{"v":0.0,"x":7,"y":17},{"v":0.0,"x":8,"y":17},{"v":13.0,"x":9,"y":17},{"v":0.0,"x":10,"y":17},{"v":0.0,"x":11,"y":17},{"v":1.0,"x":12,"y":17},{"v":2.0,"x":13,"y":17},{"v":0.0,"x":14,"y":17},{"v":2.0,"x":15,"y":17},{"v":0.0,"x":16,"y":17},{"v":0.0,"x":17,"y":17},{"v":1.0,"x":18,"y":17},{"v":2.0,"x":19,"y":17},{"v":0.0,"x":20,"y":17},{"v":206.0,"x":21,"y":17},{"v":0.0,"x":22,"y":17},{"v":3.0,"x":23,"y":17},{"v":0.0,"x":24,"y":17},{"v":0.0,"x":25,"y":17},{"v":0.0,"x":26,"y":17},{"v":0.0,"x":27,"y":17},{"v":1377.0,"x":0,"y":18},{"v":2356.0,"x":1,"y":18},{"v":41.0,"x":2,"y":18},{"v":99.0,"x":3,"y":18},{"v":187.0,"x":4,"y":18},{"v":1697.0,"x":5,"y":18},{"v":9.0,"x":6,"y":18},{"v":76.0,"x":7,"y":18},{"v":121.0,"x":8,"y":18},{"v":3033.0,"x":9,"y":18},{"v":25.0,"x":10,"y":18},{"v":90.0,"x":11,"y":18},{"v":413.0,"x":12,"y":18},{"v":162.0,"x":13,"y":18},{"v":140.0,"x":14,"y":18},{"v":869.0,"x":15,"y":18},{"v":14.0,"x":16,"y":18},{"v":16.0,"x":17,"y":18},{"v":425.0,"x":18,"y":18},{"v":190.0,"x":19,"y":18},{"v":208.0,"x":20,"y":18},{"v":252.0,"x":21,"y":18},{"v":80.0,"x":22,"y":18},{"v":21.0,"x":23,"y":18},{"v":3.0,"x":24,"y":18},{"v":773.0,"x":25,"y":18},{"v":23.0,"x":26,"y":18},{"v":0.0,"x":27,"y":18},{"v":1169.0,"x":0,"y":19},{"v":1201.0,"x":1,"y":19},{"v":21.0,"x":2,"y":19},{"v":60.0,"x":3,"y":19},{"v":9.0,"x":4,"y":19},{"v":884.0,"x":5,"y":19},{"v":2.0,"x":6,"y":19},{"v":2.0,"x":7,"y":19},{"v":1285.0,"x":8,"y":19},{"v":684.0,"x":9,"y":19},{"v":2.0,"x":10,"y":19},{"v":82.0,"x":11,"y":19},{"v":279.0,"x":12,"y":19},{"v":90.0,"x":13,"y":19},{"v":24.0,"x":14,"y":19},{"v":531.0,"x":15,"y":19},{"v":51.0,"x":16,"y":19},{"v":1.0,"x":17,"y":19},{"v":55.0,"x":18,"y":19},{"v":461.0,"x":19,"y":19},{"v":765.0,"x":20,"y":19},{"v":185.0,"x":21,"y":19},{"v":14.0,"x":22,"y":19},{"v":24.0,"x":23,"y":19},{"v":0.0,"x":24,"y":19},{"v":215.0,"x":25,"y":19},{"v":10.0,"x":26,"y":19},{"v":0.0,"x":27,"y":19},{"v":483.0,"x":0,"y":20},{"v":1027.0,"x":1,"y":20},{"v":1.0,"x":2,"y":20},{"v":17.0,"x":3,"y":20},{"v":0.0,"x":4,"y":20},{"v":716.0,"x":5,"y":20},{"v":2.0,"x":6,"y":20},{"v":2.0,"x":7,"y":20},{"v":647.0,"x":8,"y":20},{"v":532.0,"x":9,"y":20},{"v":3.0,"x":10,"y":20},{"v":0.0,"x":11,"y":20},{"v":134.0,"x":12,"y":20},{"v":4.0,"x":13,"y":20},{"v":22.0,"x":14,"y":20},{"v":667.0,"x":15,"y":20},{"v":0.0,"x":16,"y":20},{"v":0.0,"x":17,"y":20},{"v":352.0,"x":18,"y":20},{"v":35.0,"x":19,"y":20},{"v":374.0,"x":20,"y":20},{"v":78.0,"x":21,"y":20},{"v":15.0,"x":22,"y":20},{"v":11.0,"x":23,"y":20},{"v":2.0,"x":24,"y":20},{"v":341.0,"x":25,"y":20},{"v":105.0,"x":26,"y":20},{"v":0.0,"x":27,"y":20},{"v":155.0,"x":0,"y":21},{"v":163.0,"x":1,"y":21},{"v":103.0,"x":2,"y":21},{"v":103.0,"x":3,"y":21},{"v":136.0,"x":4,"y":21},{"v":169.0,"x":5,"y":21},{"v":19.0,"x":6,"y":21},{"v":47.0,"x":7,"y":21},{"v":58.0,"x":8,"y":21},{"v":121.0,"x":9,"y":21},{"v":14.0,"x":10,"y":21},{"v":93.0,"x":11,"y":21},{"v":301.0,"x":12,"y":21},{"v":154.0,"x":13,"y":21},{"v":275.0,"x":14,"y":21},{"v":10.0,"x":15,"y":21},{"v":16.0,"x":16,"y":21},{"v":10.0,"x":17,"y":21},{"v":414.0,"x":18,"y":21},{"v":474.0,"x":19,"y":21},{"v":82.0,"x":20,"y":21},{"v":3.0,"x":21,"y":21},{"v":37.0,"x":22,"y":21},{"v":86.0,"x":23,"y":21},{"v":34.0,"x":24,"y":21},{"v":13.0,"x":25,"y":21},{"v":45.0,"x":26,"y":21},{"v":0.0,"x":27,"y":21},{"v":88.0,"x":0,"y":22},{"v":642.0,"x":1,"y":22},{"v":1.0,"x":2,"y":22},{"v":0.0,"x":3,"y":22},{"v":1.0,"x":4,"y":22},{"v":568.0,"x":5,"y":22},{"v":0.0,"x":6,"y":22},{"v":0.0,"x":7,"y":22},{"v":1.0,"x":8,"y":22},{"v":911.0,"x":9,"y":22},{"v":0.0,"x":10,"y":22},{"v":3.0,"x":11,"y":22},{"v":14.0,"x":12,"y":22},{"v":0.0,"x":13,"y":22},{"v":8.0,"x":14,"y":22},{"v":153.0,"x":15,"y":22},{"v":0.0,"x":16,"y":22},{"v":0.0,"x":17,"y":22},{"v":48.0,"x":18,"y":22},{"v":0.0,"x":19,"y":22},{"v":0.0,"x":20,"y":22},{"v":7.0,"x":21,"y":22},{"v":7.0,"x":22,"y":22},{"v":0.0,"x":23,"y":22},{"v":0.0,"x":24,"y":22},{"v":121.0,"x":25,"y":22},{"v":0.0,"x":26,"y":22},{"v":0.0,"x":27,"y":22},{"v":51.0,"x":0,"y":23},{"v":280.0,"x":1,"y":23},{"v":1.0,"x":2,"y":23},{"v":0.0,"x":3,"y":23},{"v":8.0,"x":4,"y":23},{"v":149.0,"x":5,"y":23},{"v":2.0,"x":6,"y":23},{"v":1.0,"x":7,"y":23},{"v":23.0,"x":8,"y":23},{"v":148.0,"x":9,"y":23},{"v":0.0,"x":10,"y":23},{"v":6.0,"x":11,"y":23},{"v":13.0,"x":12,"y":23},{"v":2.0,"x":13,"y":23},{"v":58.0,"x":14,"y":23},{"v":36.0,"x":15,"y":23},{"v":0.0,"x":16,"y":23},{"v":0.0,"x":17,"y":23},{"v":22.0,"x":18,"y":23},{"v":20.0,"x":19,"y":23},{"v":8.0,"x":20,"y":23},{"v":25.0,"x":21,"y":23},{"v":0.0,"x":22,"y":23},{"v":2.0,"x":23,"y":23},{"v":0.0,"x":24,"y":23},{"v":73.0,"x":25,"y":23},{"v":1.0,"x":26,"y":23},{"v":0.0,"x":27,"y":23},{"v":164.0,"x":0,"y":24},{"v":103.0,"x":1,"y":24},{"v":1.0,"x":2,"y":24},{"v":4.0,"x":3,"y":24},{"v":5.0,"x":4,"y":24},{"v":36.0,"x":5,"y":24},{"v":3.0,"x":6,"y":24},{"v":0.0,"x":7,"y":24},{"v":1.0,"x":8,"y":24},{"v":102.0,"x":9,"y":24},{"v":0.0,"x":10,"y":24},{"v":0.0,"x":11,"y":24},{"v":39.0,"x":12,"y":24},{"v":1.0,"x":13,"y":24},{"v":1.0,"x":14,"y":24},{"v":41.0,"x":15,"y":24},{"v":0.0,"x":16,"y":24},{"v":0.0,"x":17,"y":24},{"v":0.0,"x":18,"y":24},{"v":31.0,"x":19,"y":24},{"v":70.0,"x":20,"y":24},{"v":5.0,"x":21,"y":24},{"v":0.0,"x":22,"y":24},{"v":3.0,"x":23,"y":24},{"v":38.0,"x":24,"y":24},{"v":30.0,"x":25,"y":24},{"v":19.0,"x":26,"y":24},{"v":0.0,"x":27,"y":24},{"v":2007.0,"x":0,"y":25},{"v":2143.0,"x":1,"y":25},{"v":27.0,"x":2,"y":25},{"v":115.0,"x":3,"y":25},{"v":272.0,"x":4,"y":25},{"v":301.0,"x":5,"y":25},{"v":12.0,"x":6,"y":25},{"v":30.0,"x":7,"y":25},{"v":22.0,"x":8,"y":25},{"v":192.0,"x":9,"y":25},{"v":23.0,"x":10,"y":25},{"v":86.0,"x":11,"y":25},{"v":1104.0,"x":12,"y":25},{"v":148.0,"x":13,"y":25},{"v":1826.0,"x":14,"y":25},{"v":271.0,"x":15,"y":25},{"v":15.0,"x":16,"y":25},{"v":6.0,"x":17,"y":25},{"v":291.0,"x":18,"y":25},{"v":401.0,"x":19,"y":25},{"v":104.0,"x":20,"y":25},{"v":141.0,"x":21,"y":25},{"v":106.0,"x":22,"y":25},{"v":4.0,"x":23,"y":25},{"v":28.0,"x":24,"y":25},{"v":23.0,"x":25,"y":25},{"v":78.0,"x":26,"y":25},{"v":0.0,"x":27,"y":25},{"v":160.0,"x":0,"y":26},{"v":860.0,"x":1,"y":26},{"v":4.0,"x":2,"y":26},{"v":2.0,"x":3,"y":26},{"v":2.0,"x":4,"y":26},{"v":373.0,"x":5,"y":26},{"v":0.0,"x":6,"y":26},{"v":1.0,"x":7,"y":26},{"v":43.0,"x":8,"y":26},{"v":364.0,"x":9,"y":26},{"v":2.0,"x":10,"y":26},{"v":2.0,"x":11,"y":26},{"v":123.0,"x":12,"y":26},{"v":35.0,"x":13,"y":26},{"v":4.0,"x":14,"y":26},{"v":110.0,"x":15,"y":26},{"v":2.0,"x":16,"y":26},{"v":0.0,"x":17,"y":26},{"v":32.0,"x":18,"y":26},{"v":4.0,"x":19,"y":26},{"v":4.0,"x":20,"y":26},{"v":73.0,"x":21,"y":26},{"v":2.0,"x":22,"y":26},{"v":3.0,"x":23,"y":26},{"v":1.0,"x":24,"y":26},{"v":147.0,"x":25,"y":26},{"v":45.0,"x":26,"y":26},{"v":0.0,"x":27,"y":26},{"v":0.0,"x":0,"y":27},{"v":0.0,"x":1,"y":27},{"v":0.0,"x":2,"y":27},{"v":0.0,"x":3,"y":27},{"v":0.0,"x":4,"y":27},{"v":0.0,"x":5,"y":27},{"v":0.0,"x":6,"y":27},{"v":0.0,"x":7,"y":27},{"v":0.0,"x":8,"y":27},{"v":0.0,"x":9,"y":27},{"v":0.0,"x":10,"y":27},{"v":0.0,"x":11,"y":27},{"v":0.0,"x":12,"y":27},{"v":0.0,"x":13,"y":27},{"v":0.0,"x":14,"y":27},{"v":0.0,"x":15,"y":27},{"v":0.0,"x":16,"y":27},{"v":0.0,"x":17,"y":27},{"v":0.0,"x":18,"y":27},{"v":0.0,"x":19,"y":27},{"v":0.0,"x":20,"y":27},{"v":0.0,"x":21,"y":27},{"v":0.0,"x":22,"y":27},{"v":0.0,"x":23,"y":27},{"v":0.0,"x":24,"y":27},{"v":0.0,"x":25,"y":27},{"v":0.0,"x":26,"y":27},{"v":0.0,"x":27,"y":27}]},"encoding":{"color":{"field":"v","legend":{"title":null},"scale":{"reverse":false,"scheme":"blues"},"type":"quantitative"},"x":{"axis":null,"field":"x","type":"ordinal"},"y":{"axis":null,"field":"y","type":"ordinal"}},"height":700,"mark":{"tooltip":true,"type":"rect"},"title":{"text":"Bigram Count"},"width":700}

Sampling The Model

The current bigram_count_tensor are the raw counts of occurrences for each bigram. This is nice for visualizing our data however this is not useful for training a model

The bigram_count_tensor can be converted into probability distributions to understand what the probability of a bigram occuring is in our dataset

p = bigram_count_tensor[0]
p = Nx.divide(p, Nx.sum(p))

#Nx.Tensor<
  f32[28]
  [0.0, 0.13767053186893463, 0.040770456194877625, 0.048137858510017395, 0.05275809392333031, 0.047794461250305176, 0.013017825782299042, 0.020884713158011436, 0.02728436328470707, 0.018449723720550537, 0.07560952752828598, 0.09249836206436157, 0.04907439276576042, 0.07923079282045364, 0.03577560558915138, 0.012299816124141216, 0.01607717014849186, 0.0028720381669700146, 0.05116598680615425, 0.06415259093046188, 0.040832892060279846, 0.002434988971799612, 0.01173789519816637, 0.009583866223692894, 0.0041831862181425095, 0.016701526939868927, 0.02900134213268757, 0.0]
>

key = Nx.Random.key(2_147_483_647)
{p, _} = Nx.Random.uniform(key, 0, 1, shape: {3})
p = Nx.divide(p, Nx.sum(p))
Makemore.multinomial(p, key, samples: 100, replace: true)

#Nx.Tensor<
  s64[100]
  [2, 0, 0, 2, 0, 0, 2, 2, 1, 1, 1, 2, 1, 0, 1, 1, 2, 0, 0, 1, 2, 2, 2, 0, 1, 2, 1, 1, 0, 1, 2, 1, 1, 1, 0, 2, 2, 2, 1, 2, 0, 1, 1, 0, 1, 1, 1, 0, 1, 2, ...]
>

TODO: Sampling from model

https://youtu.be/PaCmpygFfXo?t=2001

Enum.reduce_while(1..1000, {0, ""}, fn _, {ix, word} ->
  p = bigram_count_tensor[ix]
  p = Nx.divide(p, Nx.sum(p))
  key = Nx.Random.key(2_147_483_647)
  new_ix = Makemore.multinomial(p, key, samples: 1, replace: true) |> Nx.to_list() |> List.first()
  char = Map.fetch!(itos, new_ix)

  cond do
    new_ix == 0 ->
      {:halt, {new_ix, "#{word}#{char}"}}

    true ->
      {:cont, {new_ix, "#{word}#{char}"}}
  end
end)

{15,
 "oooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo"}