Powered by AppSignal & Oban Pro
Would you like to see your link here? Contact us
Notesclub

Generate Advent Of Code solutions with LLM

code_generation_with_llm.livemd

Generate Advent Of Code solutions with LLM

Mix.install([
  :req,
  :json
])

Load all advent of code from hugging face

# dataset_url = "https://huggingface.co/datasets/isavita/advent-of-code/resolve/main/train.json"
# dataset = Req.get!(dataset_url, receive_timeout: 600_000).body
dataset = File.read!("/Users/isavita/git/advent-of-code/train.json") |> Jason.decode!()
defmodule Constants do
  def file_ext, do: ".zig"
  def lang, do: "zig"
  def compiler, do: "/opt/homebrew/bin/zig"
  def runtime, do: "/opt/homebrew/bin/zig"
end

Constants.runtime()
program = ~S|
const std = @import("std");

pub fn main() void {
    std.debug.print("Hello, World!\n", .{});
}
|

File.write!("test#{Constants.file_ext()}", program)
# System.cmd(Constants.compiler(), ["task#{Constants.file_ext()}"], stderr_to_stdout: true)
# System.cmd("chmod", ["+x", "test#{Constants.file_ext()}"])
# System.cmd("#{File.cwd!()}/test", [], stderr_to_stdout: true)
# "#{File.cwd!()}/test#{Constants.file_ext()}"
# 
# System.cmd("/Users/isavita/test.sh", [])
# System.cmd(Constants.runtime(), ["script", "test#{Constants.file_ext()}"], stderr_to_stdout: true)
System.cmd(Constants.runtime(), ["run", "test#{Constants.file_ext()}"], stderr_to_stdout: true)
solutions = File.ls!("#{File.cwd!()}/code/advent_generated/zig")
solved_parts = Enum.map(solutions, &String.replace(&1, Constants.file_ext(), ""))

dataset_202X =
  Enum.filter(dataset, fn part ->
    part["solution_lang"] == "go" && part["name"] not in solved_parts
  end)

Manage Request to Mistral API

defmodule MistralClient do
  # 10 mins
  @timeout 600_000
  @url "https://api.mistral.ai/v1/chat/completions"
  @system_prompt """
  You are an expert programmer that writes simple, concise code and no comments or explanation.
  Write a elixir module that it reads its input from a file "input.txt" and solve the following task.
  The module should have only one public function called `call` with arity 0.
  The answer should be RETURN instead of printed in the stdout.
  TASK:
  """
  def call(model, task, system_prompt \\ @system_prompt) do
    prompt = "#{system_prompt}\n#{task}"

    payload =
      %{
        "model" => model,
        "messages" => [
          %{"role" => "user", "content" => prompt}
        ],
        "temperature" => 0.7,
        "stream" => false
      }
      |> Jason.encode!()

    Req.post!(@url, body: payload, headers: headers(), receive_timeout: @timeout).body
    |> Map.get("choices", [%{}])
    |> hd()
    |> get_in(["message", "content"])
  end

  defp headers do
    [
      {"Content-Type", "application/json"},
      {"Accept", "application/json"},
      {"Authorization", "Bearer " <> mistral_api_key()}
    ]
  end

  defp mistral_api_key, do: System.fetch_env!("LB_MISTRAL_API_KEY")
end
defmodule OpenAIClient do
  # 12 mins
  @timeout 1_200_000
  @url "https://api.openai.com/v1/chat/completions"
  @system_prompt ~s|You are a highly experienced programmer with a PhD in computer science participating in a coding challenge.
Write clean, efficient code without unnecessary comments, demonstrating your advanced skills by solving problems practically and concisely.
Aim to produce optimal and concise solutions, leveraging your decade of industry experience.|

  def call(model, prompt, system_prompt \\ @system_prompt) do
    payload =
      %{
        "model" => model,
        "temperature" => 0.5,
        "messages" => [
          %{"role" => "system", "content" => system_prompt},
          %{"role" => "user", "content" => "#{system_prompt}\n#{prompt}"}
        ]
      }
      |> Jason.encode!()

    Req.post!(@url, body: payload, headers: headers(), receive_timeout: @timeout).body
    |> Map.get("choices", [%{}])
    |> hd()
    |> get_in(["message", "content"])
  end

  defp headers do
    [
      {"authorization", "Bearer #{openai_api_key()}"},
      {"content-type", "application/json"},
      {"accept", "application/json"}
    ]
  end

  defp openai_api_key, do: System.fetch_env!("LB_OPENAI_API_KEY")
end
defmodule AnthropicClient do
  # 10 mins
  @timeout 600_000
  @url "https://api.anthropic.com/v1/messages"
  @system_prompt """
  As a highly experienced software developer with a degree in computer science and a background in competitive programming, strive to:

  1. Write efficient and concise solutions.
  2. Prioritize clarity and readability in your code.
  3. Include only necessary comments, focusing on explaining any tricky or complex parts of the code.
  4. Ensure your code is well-organized and easy to follow.
  5. Use appropriate data structures and algorithms for the problem at hand.
  6. Follow best practices and coding standards for the language you are using.

  By following these guidelines, you can write high-quality code that is both efficient and maintainable.
  """
  def call(model, task, system_prompt \\ @system_prompt) do
    prompt = "#{system_prompt}#{task}"

    payload = %{
      "model" => model,
      "max_tokens" => 4096,
      "messages" => [
        %{"role" => "user", "content" => prompt}
      ],
      "temperature" => 0.1,
      "stream" => false
    }

    case Req.post(@url, json: payload, headers: headers(), receive_timeout: @timeout) do
      {:ok, resp} ->
        if resp.status == 200 do
          body = resp.body
          content = hd(body["content"] || [%{}])
          content["text"]
        else
          IO.puts("#{__MODULE__}: #{inspect(resp.body)}")
          ""
        end

      error ->
        IO.puts("#{__MODULE__}: #{inspect(error)}")
        ""
    end
  end

  defp headers do
    %{
      "content-type" => "application/json",
      "anthropic-version" => "2023-06-01",
      "x-api-key" => System.fetch_env!("LB_ANTHROPIC_API_KEY")
    }
  end
end
defmodule RunWithTimeout do
  def call(fun, timeout) do
    task =
      Task.async(fn ->
        try do
          # Attempt to run the function
          result = fun.()
          {:ok, result}
        rescue
          exception ->
            # If an exception occurs, return an error tuple
            {:error, Exception.message(exception)}
        end
      end)

    try do
      Task.await(task, timeout)
    rescue
      # Catches exceptions and converts them to a tuple
      exception ->
        {:error, Exception.message(exception)}
    catch
      # Catches exits such as those from Task.await timeout
      :exit, _ ->
        # Ensures that the task is not left running
        Task.shutdown(task, :brutal_kill)

        # System.cmd("pgrep", ["-f", "test#{Constants.file_ext()}"], stderr_to_stdout: true)
        {output, _} =
          System.cmd("pgrep", ["-f", "test"], stderr_to_stdout: true)
          |> IO.inspect(label: "kill")

        kill_os_process(output) |> IO.inspect(label: "kill")
        {:error, "The operation timed out."}

      # Catches throws, which are non-standard in Elixir but still possible
      :throw, value ->
        {:error, "The operation was aborted: #{inspect(value)}"}
    end
  end

  defp kill_os_process(output) do
    Enum.each(String.split(output, "\n"), fn pid ->
      # Ignore empty lines
      if pid != "" do
        # Kill each process found
        System.cmd("kill", [pid])
      else
        :ok
      end
    end)
  end
end

Manage Request to local ollama model

defmodule OllamaClient do
  # 20 mins
  @timeout 1_200_000
  # "http://localhost:11434/api/generate"
  @url "http://localhost:11434/v1/chat/completions"
  @sys_prompt """
  As a highly experienced software developer with a degree in computer science and a background in competitive programming, strive to:

  1. Write efficient and concise solutions.
  2. Prioritize clarity and readability in your code.
  3. Include only necessary comments, focusing on explaining any tricky or complex parts of the code.
  4. Ensure your code is well-organized and easy to follow.
  5. Use appropriate data structures and algorithms for the problem at hand.
  6. Follow best practices and coding standards for the language you are using.

  By following these guidelines, you can write high-quality code that is both efficient and maintainable.
  """
  @system_prompt """
  You are #{Constants.lang()} programmer that writes simple, concise code and no comments or explanation.
  Write #{Constants.lang()} program that it reads its input from a file "input.txt" and solve the following task.
  The answer MUST BE printed to the stdout.
  TASK:
  """
  def call(model, task, system_prompt \\ @system_prompt) do
    prompt = "#{system_prompt}#{task}"

    payload =
      Jason.encode!(%{
        "model" => model,
        "messages" => [
          # %{
          #   "role" => "system",
          #   "content" => @sys_prompt
          # },
          %{
            "role" => "user",
            "content" => prompt
          }
        ],
        "stream" => false
      })

    case Req.post(@url, body: payload, receive_timeout: @timeout) do
      {:ok, resp} ->
        body = resp.body
        choice = hd(body["choices"] || [%{}])
        choice["message"]["content"]

      error ->
        IO.puts("#{__MODULE__}: #{inspect(error)}")
        ""
    end
  end
end
# resp = OllamaClient.call("neural-chat", "Write me a Nim program that reads a number from a file input.txt and multiply it by itself and prints the answer to the stdout.", "")

Code evaluator

defmodule EvaluateCode do
  def call(code, file_content) do
    try do
      File.write!("test#{Constants.file_ext()}", code |> IO.inspect(label: "code"))
      File.write!("input.txt", file_content)

      # System.cmd(Constants.compiler(), ["test#{Constants.file_ext()}"], stderr_to_stdout: true)

      # System.cmd(Constants.compiler(), ["-framework", "Foundation", "test#{Constants.file_ext()}", "-o", "test"]) |> IO.inspect(label: 1)
      # System.cmd("chmod", ["+x", "test#{Constants.file_ext()}"]) |> IO.inspect(label: 2)

      {:ok, {result, exit_status}} =
        RunWithTimeout.call(
          fn ->
            # System.cmd("#{File.cwd!()}/test", [], stderr_to_stdout: true)
            # args = ["run", "--rm", "--platform", "linux/amd64", "-i", "-v", "#{File.cwd!()}/test.d:/test.d", "-v", "#{File.cwd!()}/input.txt:/input.txt", "dlang2/dmd-ubuntu", "/bin/bash", "-c", "dmd test.d && ./test"]
            args = ["run", "test#{Constants.file_ext()}"]

            System.cmd(Constants.runtime(), args, stderr_to_stdout: true)
            |> IO.inspect(label: "run")
          end,
          60_000
        )
        |> IO.inspect(label: "here")

      if exit_status == 0 do
        {:ok, inspect(result)}
      else
        {:error, inspect(result)}
      end
    rescue
      exception ->
        {:error, Exception.message(exception)}
    end
  end
end

Add mutiple time FunctionRunner

defmodule FunctionRunner do
  def run_max_times(function, max_attempts) do
    do_run_max_times(function, max_attempts, 1)
  end

  defp do_run_max_times(function, max_attempts, attempt) when attempt <= max_attempts do
    case function.() do
      :ok -> :ok
      _ -> do_run_max_times(function, max_attempts, attempt + 1)
    end
  end

  defp do_run_max_times(_function, _max_attempts, _attempt), do: :error
end

defmodule CodeResponseSanitizer do
  def call(input) when is_binary(input) do
    case Regex.scan(~r/```(?:#{Constants.lang()})?(.*?)```/sim, input) do
      [[_full_match, code] | _tail] -> code
      _ -> input |> IO.inspect(label: "NO MATCH")
    end
  end
end

defmodule TaskSolver do
  @examples """
  Here are a few examples of #{Constants.lang()} language solutions for some coding challenge problems:
  1. Solution for "Day 1: Calculate the final floor" of Part One of the challenge:
  ```#{Constants.lang()}
  const std = @import("std");

  pub fn main() !void {
      var file = try std.fs.cwd().openFile("input.txt", .{});
      defer file.close();

      var buf_reader = std.io.bufferedReader(file.reader());
      var in_stream = buf_reader.reader();

      var floor: i32 = 0;

      while (true) {
          const char = in_stream.readByte() catch |err| switch (err) {
              error.EndOfStream => break,
              else => return err,
          };

          switch (char) {
              '(' => floor += 1,
              ')' => floor -= 1,
              else => unreachable,
          }
      }

      std.debug.print("Final floor: {}\n", .{floor});
  }
  ```
  2. Solution for "Day 2: Rock Paper Scissors" of the Part One of the challenge:
  ```#{Constants.lang()}
  const std = @import("std");

  pub fn main() !void {
      var file = try std.fs.cwd().openFile("input.txt", .{});
      defer file.close();

      var reader = std.io.bufferedReader(file.reader());
      var line_reader = reader.reader();

      var total_score: i32 = 0;

      var buffer: [1024]u8 = undefined;
      while (try line_reader.readUntilDelimiterOrEof(&buffer, '\n')) |line| {
          const opponent = line[0];
          const your_move = line[2];

          var score: i32 = 0;
          if (your_move == 'X') {
              score = 1;
          } else if (your_move == 'Y') {
              score = 2;
          } else if (your_move == 'Z') {
              score = 3;
          }

          if ((opponent == 'A' and your_move == 'Y') or (opponent == 'B' and your_move == 'Z') or (opponent == 'C' and your_move == 'X')) {
              score += 6;
          } else if ((opponent == 'A' and your_move == 'X') or (opponent == 'B' and your_move == 'Y') or (opponent == 'C' and your_move == 'Z')) {
              score += 3;
          }

          total_score += score;
      }

      std.debug.print("{}\n", .{total_score});
  }
  ```
  3. Solution for "Day 18: Boiling Boulders" Part Two of the challenge:
  ```#{Constants.lang()}
  const std = @import("std");

  const Pt3 = struct {
      x: i32,
      y: i32,
      z: i32,
  };

  fn addPt3(p1: Pt3, p2: Pt3) Pt3 {
      return Pt3{ .x = p1.x + p2.x, .y = p1.y + p2.y, .z = p1.z + p2.z };
  }

  fn minInt(a: i32, b: i32) i32 {
      return if (a < b) a else b;
  }

  fn maxInt(a: i32, b: i32) i32 {
      return if (a > b) a else b;
  }

  pub fn main() !void {
      var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
      defer arena.deinit();

      const allocator = arena.allocator();

      var cubes = std.AutoHashMap(Pt3, void).init(allocator);
      defer cubes.deinit();

      const neighbors = [6]Pt3{
          Pt3{ .x = -1, .y = 0, .z = 0 },
          Pt3{ .x = 1, .y = 0, .z = 0 },
          Pt3{ .x = 0, .y = -1, .z = 0 },
          Pt3{ .x = 0, .y = 1, .z = 0 },
          Pt3{ .x = 0, .y = 0, .z = -1 },
          Pt3{ .x = 0, .y = 0, .z = 1 },
      };

      var min_pt = Pt3{ .x = std.math.maxInt(i32), .y = std.math.maxInt(i32), .z = std.math.maxInt(i32) };
      var max_pt = Pt3{ .x = std.math.minInt(i32), .y = std.math.minInt(i32), .z = std.math.minInt(i32) };

      var file = try std.fs.cwd().openFile("input.txt", .{});
      defer file.close();

      var reader = file.reader();
      var buf: [1024]u8 = undefined;
      var line_reader = std.io.bufferedReader(reader);
      var in = line_reader.reader();

      while (try in.readUntilDelimiterOrEof(buf[0..], '\n')) |line| {
          if (line.len == 0) continue;

          var parts = std.mem.split(u8, line, ",");
          const x = try std.fmt.parseInt(i32, parts.next().?, 10);
          const y = try std.fmt.parseInt(i32, parts.next().?, 10);
          const z = try std.fmt.parseInt(i32, parts.next().?, 10);

          const cube = Pt3{ .x = x, .y = y, .z = z };
          try cubes.put(cube, {});

          min_pt = Pt3{
              .x = minInt(min_pt.x, cube.x),
              .y = minInt(min_pt.y, cube.y),
              .z = minInt(min_pt.z, cube.z),
          };
          max_pt = Pt3{
              .x = maxInt(max_pt.x, cube.x),
              .y = maxInt(max_pt.y, cube.y),
              .z = maxInt(max_pt.z, cube.z),
          };
      }

      min_pt = addPt3(min_pt, Pt3{ .x = -1, .y = -1, .z = -1 });
      max_pt = addPt3(max_pt, Pt3{ .x = 1, .y = 1, .z = 1 });

      var faces: usize = 0;
      var queue = std.ArrayList(Pt3).init(allocator);
      defer queue.deinit();

      var seen = std.AutoHashMap(Pt3, void).init(allocator);
      defer seen.deinit();

      try seen.put(min_pt, {});
      try queue.append(min_pt);

      while (queue.items.len > 0) {
          const curr = queue.pop();

          for (neighbors) |delta| {
              const next = addPt3(curr, delta);
              if (next.x < min_pt.x or
                  next.y < min_pt.y or
                  next.z < min_pt.z or
                  next.x > max_pt.x or
                  next.y > max_pt.y or
                  next.z > max_pt.z)
              {
                  continue;
              }

              if (cubes.contains(next)) {
                  faces += 1;
              } else if (!seen.contains(next)) {
                  try seen.put(next, {});
                  try queue.append(next);
              }
          }
      }

      std.debug.print("{d}\n", .{faces});
  }
  ```
  """
  def call(day, model \\ "gpt-3.5-turbo-0125") do
    # task = Map.fetch!(day, "task")
    input = Map.fetch!(day, "input")
    answer = Map.fetch!(day, "answer")
    solution = Map.fetch!(day, "solution")

    # system_prompt = """
    # Write an #{Constants.lang()} program that reads input from a file called input.txt and prints the output to standard output.
    # Focus on writing clean, efficient code that demonstrates your programming skills by concisely solving the challenge.

    # Coding challenge:
    # """

    system_prompt = """
    You are an expert programmer that writes simple, concise code and no comments or explanation.
    Given this golang program write a #{Constants.lang()} version that it reads its input from a file "input.txt" and solve the following task.
    The program should print the answer.

    Here are few examples on previous solutions of challenges in #{Constants.lang()}:
    #{@examples}

    Golang Solution:
    """

    path =
      ~s|#{System.get_env("HOME")}/code/advent_generated/#{day["name"]}#{Constants.file_ext()}|

    code = generate_solution(model, solution, system_prompt)

    with {:ok, result} <- EvaluateCode.call(code, input),
         true <- valid_solution?(result, answer) do
      File.write!(path, code)
      {:ok, path}
    else
      {:error, err} ->
        handle_error(model, code, err, input, answer, path)

      error ->
        error
    end
  end

  defp generate_solution(model, propmt, system_prompt) do
    # ft:gpt-3.5-turbo-0125:personal:erlang-0303:8yge7H3W
    # "ft:gpt-3.5-turbo-0125:personal:elixir:90xUsEr3"
    # "claude-3-haiku-20240307"

    # MistralClient.call(model, propmt, system_prompt)
    # OllamaClient.call(model, propmt, system_prompt)
    # AnthropicClient.call(model, propmt, system_prompt)
    (OpenAIClient.call(model, propmt, system_prompt) ||
       "")
    |> IO.inspect(label: "before parse")
    |> CodeResponseSanitizer.call()
    |> String.trim()
  end

  defp handle_error(model, code, err, input, answer, path) do
    prompt =
      """
      Your previous solution produced following error:
      ```#{Constants.lang()}
      #{code}
      ```
      Produce following error:
      #{err}

      Give me full fixed solution without missing any code.
      """
      |> IO.inspect(label: "NEW PROMT")

    system_prompt = """
    You are an expert in #{Constants.lang()}, skilled at analyzing provided code alongside its error messages.
    Examine the issues and return a fully working code solution that addresses and resolves all identified problems.
    """

    code = generate_solution(model, prompt, system_prompt) |> IO.inspect(label: "NEXT TRY")

    with {:ok, result} <- EvaluateCode.call(code, input) |> IO.inspect(label: "NEXT call"),
         true <- valid_solution?(result, answer) |> IO.inspect(label: "NEXT valid check") do
      File.write!(path, code) |> IO.inspect(label: "it uses this")
      {:ok, path}
    else
      error -> {:error, error}
    end
  end

  defp valid_solution?(result, answer) do
    # day8_part2_2016
    # day10_part1_2018
    # day1_part1_2019
    # day8_part2_2019
    # day11_part2_2019
    # day13_part2_2021
    # day10_part2_2022
    String.contains?(result, answer) ||
      String.contains?(result, [
        ".##..####.###..#..#.###..####.###....##.###...###.",
        " ##  #### ###  #  # ###  #### ###    ## ###   ### "
      ]) ||
      String.contains?(result, [
        "#....#..#....#.....###..######....##....#....#....##....######",
        "#    #  #    #     ###  ######    ##    #    #    ##    ######"
      ]) ||
      String.contains?(result, ["3.465154e+06", "3.465154e+6"]) ||
      String.contains?(result, [
        "####.###..####.#..#.###..\n#....#..#....#.#..#.#..#.",
        "#### ###  #### #  # ###  \n#    #  #    # #  # #  # "
      ]) ||
      String.contains?(result, [
        ".#....###....##.#..#.####.#..#.#....#..#.\n",
        " #    ###    ## #  # #### #  # #    #  # \n",
        " █    ███    ██ █  █ ████ █  █ █    █  █ \n"
      ]) ||
      String.contains?(result, [
        "#..#.#..#.#..#.#..#.#..#.#..#.#..#....#",
        "#  # #  # #  # #  # #  # #  # #  #    #"
      ]) ||
      String.contains?(result, [
        "###..###..###...##..###...##...##..####.",
        "###  ###  ###   ##  ###   ##   ##  #### "
      ])
  end
end
len = length(dataset_202X)

Enum.reduce(dataset_202X, 1, fn day, count ->
  IO.puts(day["name"])

  FunctionRunner.run_max_times(
    # growthwtf/hermes2pro7b, "claude-3-haiku-20240307"
    fn -> TaskSolver.call(day, "gpt-4-turbo-preview") end,
    1
  )

  :io.format("~.1f%\n", [count / len * 100])
  count + 1
end)