Evaluating MixTCP

Mix.install([
  {:kino_vega_lite, "~> 0.1.7"},
  {:jason, "~> 1.4"},
  {:statistex, "~> 1.0"},
  {:explorer, "~> 0.7.1"}
])

alias Explorer.{DataFrame, Series}

Setup

test_runs_folder = "/Users/arnold/Projects/elixir/mix_tcp/test_runs/ash_paper_trail"

server = :tcp@localhost
Node.set_cookie(:"1234")
Node.connect(server)

Helper Functions

parse_test_run = fn test_run ->
  :erpc.call(server, MixTcp.TestRunParser, :parse, [test_run])
end

tcp = fn how_many_cycles ->
  :erpc.call(server, MixTcp.Server, :tcp, [test_runs_folder, how_many_cycles])
end

napfd = fn testcases, faults, fault_exposed_by_testcase_fn ->
  n = length(testcases)

  faults_with_index =
    faults
    |> Stream.map(fn fault ->
      testcases
      |> Enum.find_index(&fault_exposed_by_testcase_fn.(fault, &1))
      |> then(&{fault, &1})
    end)
    |> Stream.filter(fn {_fault, idx} -> idx != nil end)
    |> Enum.into(%{})

  m = Enum.count(faults)
  p = map_size(faults_with_index) / m

  if m != 0 and n != 0 do
    sum =
      faults_with_index
      |> Map.values()
      |> Enum.sum()

    p - sum / (m * n) + p / (2 * n)
  else
    0.0
  end
end

Preparing data

test_runs_per_cycles_stream =
  test_runs_folder
  |> File.ls!()
  |> Enum.sort()
  |> Stream.map(&Path.join(test_runs_folder, &1))
  |> Stream.map(&File.read!/1)
  |> Stream.map(parse_test_run)
  |> Stream.with_index()
  |> Stream.flat_map(fn {test_runs, cycle} ->
    Enum.map(test_runs, &Map.put(&1, :cycle, cycle))
  end)

Enum.frequencies_by(test_runs_per_cycles_stream, & &1.cycle)

id_by_testcase =
  test_runs_per_cycles_stream
  |> Stream.map(&{{&1.test_file, &1.line}, &1.id})
  |> Stream.uniq()
  |> Enum.into(%{})

faults =
  test_runs_per_cycles_stream
  |> Stream.filter(& &1.fail?)
  |> Stream.map(& &1.id)
  |> Stream.uniq()
  |> Enum.to_list()

cycles_to_evaluate = 0..7
tests_count_for_napfd = 30

napfd_per_run =
  cycles_to_evaluate
  |> Enum.map(fn cycles_count_to_predict_from ->
    testcases =
      test_runs_per_cycles_stream
      |> Stream.filter(&amp;(&amp;1.cycle < cycles_count_to_predict_from))
      |> Stream.map(&amp; &amp;1.id)
      |> Stream.uniq()
      |> Enum.to_list()

    verdict_per_id =
      test_runs_per_cycles_stream
      |> Stream.filter(&amp;(&amp;1.cycle == cycles_count_to_predict_from))
      |> Enum.group_by(&amp; &amp;1.id, &amp; &amp;1.fail?)
      |> Enum.into(%{}, fn {key, fails} ->
        {key, Enum.any?(fails, &amp; &amp;1)}
      end)

    neutron_napfd =
      cycles_count_to_predict_from
      |> tcp.()
      |> Enum.take(tests_count_for_napfd)
      |> Enum.map(&amp;id_by_testcase[&amp;1])
      |> napfd.(faults, fn
        id, id -> verdict_per_id[id]
        _, _ -> false
      end)

    random_napfd =
      1..100
      |> Enum.map(fn _ ->
        testcases
        |> Enum.shuffle()
        |> Enum.take(tests_count_for_napfd)
        |> napfd.(faults, fn
          id, id -> verdict_per_id[id]
          _, _ -> false
        end)
      end)
      |> Statistex.average()

    %{
      cycle: cycles_count_to_predict_from,
      neutron: neutron_napfd,
      random: random_napfd
    }
  end)

Visualization

data =
  napfd_per_run
  |> Enum.drop(1)
  |> Enum.flat_map(fn %{cycle: cycle, random: random_napfd, neutron: neutron_napfd} ->
    [
      %{"Cycle" => cycle, "NAPFD" => random_napfd, "Method" => "Random", "position" => 1},
      %{"Cycle" => cycle, "NAPFD" => neutron_napfd, "Method" => "NEUTRON", "position" => 2}
    ]
  end)

VegaLite.new()
|> VegaLite.data_from_values(data)
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "Cycle", type: :ordinal)
|> VegaLite.encode_field(:y, "NAPFD", type: :quantitative)
|> VegaLite.encode_field(:color, "Method")
|> VegaLite.encode_field(:x_offset, "position")