Generate Advent Of Code solutions with LLM
Mix.install([
:req,
:json
])
Load all advent of code from hugging face
# dataset_url = "https://huggingface.co/datasets/isavita/advent-of-code/resolve/main/train.json"
# dataset = Req.get!(dataset_url, receive_timeout: 600_000).body
dataset = File.read!("/Users/isavita/git/advent-of-code/train.json") |> Jason.decode!()
defmodule Constants do
def file_ext, do: ".clj"
def lang, do: "clojure"
def compiler, do: "/Users/isavita/build.sh"
def runtime, do: "/opt/homebrew/bin/clojure"
end
Constants.runtime()
program = ~S|
(println "Hello, World")
|
File.write!("test#{Constants.file_ext()}", program)
# System.cmd("/bin/chmod", ["777", "/Users/isavita/build.sh"])
# System.cmd(Constants.compiler(), [], stderr_to_stdout: true)
# new_path = "#{System.get_env("PATH")}:/opt/homebrew/bin/"
# System.cmd(Constants.compiler(), ["-framework", "Foundation", "-o", "test", "test#{Constants.file_ext()}"], stderr_to_stdout: true)
# System.cmd("/bin/chmod", ["755", "test#{Constants.file_ext()}"])
# System.cmd("#{File.cwd!()}/test", [], stderr_to_stdout: true)
# "#{File.cwd!()}/test#{Constants.file_ext()}"
# current_path = System.get_env("PATH")
# # Specify the directory where the 'node' binary can be found (not the path to the 'coffee' binary itself)
# node_path = "/Users/isavita/.nvm/versions/node/v19.8.1/bin:#{System.get_env("PATH")}"
# # # Append the node_bin_path to the existing PATH
# new_path = "#{node_bin_path}:#{current_path}"
# System.cmd("/bin/bash", ["test.sh"])
# # System.cmd(Constants.runtime(), ["dotnet", "run", "--project", "test"], stderr_to_stdout: true)
# System.cmd(Constants.runtime(), ["-noshell", "-s", "task", "call", "-s", "init", "stop"], stderr_to_stdout: true)
# args = [
# "-c",
# "/usr/local/bin/docker run --rm -v /Users/isavita:/app dlang2/dmd-ubuntu:latest bash -c \"dmd -run /app/test.d\""
# ]
args = ["-M", "test#{Constants.file_ext()}"]
System.cmd(Constants.runtime(), args,
stderr_to_stdout: true,
env: [{"LC_ALL", "en_US.UTF-8"}, {"LANG", "en_US.UTF-8"}, {"GUILE_AUTO_COMPILE", "0"}]
)
solutions = File.ls!("#{File.cwd!()}/code/advent_generated/clojure")
solved_parts = Enum.map(solutions, &String.replace(&1, Constants.file_ext(), ""))
dataset_202X =
Enum.filter(dataset, fn part ->
part["solution_lang"] == "go" && part["name"] not in solved_parts
end)
Manage Request to Mistral API
defmodule MistralClient do
# 10 mins
@timeout 600_000
@url "https://api.mistral.ai/v1/chat/completions"
@system_prompt """
You are an expert programmer that writes simple, concise code and no comments or explanation.
Write a elixir module that it reads its input from a file "input.txt" and solve the following task.
The module should have only one public function called `call` with arity 0.
The answer should be RETURN instead of printed in the stdout.
TASK:
"""
def call(model, task, system_prompt \\ @system_prompt) do
prompt = "#{system_prompt}\n#{task}"
payload =
%{
"model" => model,
"messages" => [
%{"role" => "user", "content" => prompt}
],
"temperature" => 0.5,
"stream" => false
}
|> Jason.encode!()
Req.post!(@url, body: payload, headers: headers(), receive_timeout: @timeout).body
|> Map.get("choices", [%{}])
|> hd()
|> get_in(["message", "content"])
end
defp headers do
[
{"Content-Type", "application/json"},
{"Accept", "application/json"},
{"Authorization", "Bearer " <> mistral_api_key()}
]
end
defp mistral_api_key, do: System.fetch_env!("LB_MISTRAL_API_KEY")
end
defmodule OpenAIClient do
# 12 mins
@timeout 1_200_000
@url "https://api.openai.com/v1/chat/completions"
@system_prompt ~s|You are a highly experienced programmer with a PhD in computer science participating in a coding challenge.
Write clean, efficient code without unnecessary comments, demonstrating your advanced skills by solving problems practically and concisely.
Aim to produce optimal and concise solutions, leveraging your decade of industry experience.|
def call(model, prompt, system_prompt \\ @system_prompt) do
sys_prompt =
"You are a highly experienced programmer with a PhD in computer science participating in a coding challenge.\nWrite clean, efficient code without unnecessary comments, demonstrating your advanced skills by solving problems practically and concisely.\nAim to produce optimal and concise solutions, leveraging your decade of industry experience."
payload =
%{
"model" => model,
"temperature" => 0.5,
"messages" => [
%{"role" => "system", "content" => sys_prompt},
%{"role" => "user", "content" => "#{system_prompt}\n#{prompt}"}
]
}
|> Jason.encode!()
Req.post!(@url, body: payload, headers: headers(), receive_timeout: @timeout).body
|> Map.get("choices", [%{}])
|> hd()
|> get_in(["message", "content"])
end
defp headers do
[
{"authorization", "Bearer #{openai_api_key()}"},
{"content-type", "application/json"},
{"accept", "application/json"}
]
end
defp openai_api_key, do: System.fetch_env!("LB_OPENAI_API_FT_KEY" || "LB_OPENAI_API_KEY")
end
defmodule AnthropicClient do
# 10 mins
@timeout 600_000
@url "https://api.anthropic.com/v1/messages"
@system_prompt """
As a highly experienced software developer with a degree in computer science and a background in competitive programming, strive to:
1. Write efficient and concise solutions.
2. Prioritize clarity and readability in your code.
3. Include only necessary comments, focusing on explaining any tricky or complex parts of the code.
4. Ensure your code is well-organized and easy to follow.
5. Use appropriate data structures and algorithms for the problem at hand.
6. Follow best practices and coding standards for the language you are using.
By following these guidelines, you can write high-quality code that is both efficient and maintainable.
"""
def call(model, task, system_prompt \\ @system_prompt) do
prompt = "#{system_prompt}#{task}"
payload = %{
"model" => model,
"max_tokens" => 4096,
"messages" => [
%{"role" => "user", "content" => prompt}
],
"temperature" => 0.5,
"stream" => false
}
case Req.post(@url, json: payload, headers: headers(), receive_timeout: @timeout) do
{:ok, resp} ->
if resp.status == 200 do
body = resp.body
content = hd(body["content"] || [%{}])
content["text"]
else
IO.puts("#{__MODULE__}: #{inspect(resp.body)}")
""
end
error ->
IO.puts("#{__MODULE__}: #{inspect(error)}")
""
end
end
defp headers do
%{
"content-type" => "application/json",
"anthropic-version" => "2023-06-01",
"x-api-key" => System.fetch_env!("LB_ANTHROPIC_API_KEY")
}
end
end
defmodule GroqClient do
# 5 mins
@timeout 300_000
@url "https://api.groq.com/openai/v1/chat/completions"
@system_prompt ~s|You are a highly experienced programmer with a PhD in computer science participating in a coding challenge.
Write clean, efficient code without unnecessary comments, demonstrating your advanced skills by solving problems practically and concisely.
Aim to produce optimal and concise solutions, leveraging your decade of industry experience.|
def call(model, prompt, system_prompt \\ @system_prompt) do
payload =
%{
"messages" => [
%{
"role" => "system",
"content" => system_prompt
},
%{
"role" => "user",
"content" => prompt
}
],
"model" => model,
"temperature" => 0.7,
"stream" => false
}
|> Jason.encode!()
resp = Req.post!(@url, body: payload, headers: headers(), receive_timeout: @timeout)
if resp.status == 200 do
resp.body
|> Map.get("choices", [%{}])
|> hd()
|> get_in(["message", "content"])
else
"please try again"
end
end
defp headers do
[
{"authorization", "Bearer #{groq_api_key()}"},
{"content-type", "application/json"},
{"accept", "application/json"}
]
end
defp groq_api_key, do: System.fetch_env!("LB_GROQ_API_KEY")
end
defmodule RunWithTimeout do
def call(fun, timeout) do
task =
Task.async(fn ->
try do
# Attempt to run the function
result = fun.()
{:ok, result}
rescue
exception ->
# If an exception occurs, return an error tuple
{:error, Exception.message(exception)}
end
end)
try do
Task.await(task, timeout)
rescue
# Catches exceptions and converts them to a tuple
exception ->
{:error, Exception.message(exception)}
catch
# Catches exits such as those from Task.await timeout
:exit, _ ->
# Ensures that the task is not left running
Task.shutdown(task, :brutal_kill)
# System.cmd("pgrep", ["-f", "test#{Constants.file_ext()}"], stderr_to_stdout: true)
{output, _} =
System.cmd("pgrep", ["-f", "test"], stderr_to_stdout: true)
|> IO.inspect(label: "kill")
kill_os_process(output) |> IO.inspect(label: "kill")
{:error, "The operation timed out."}
# Catches throws, which are non-standard in Elixir but still possible
:throw, value ->
{:error, "The operation was aborted: #{inspect(value)}"}
end
end
defp kill_os_process(output) do
Enum.each(String.split(output, "\n"), fn pid ->
# Ignore empty lines
if pid != "" do
# Kill each process found
System.cmd("kill", [pid])
else
:ok
end
end)
end
end
Manage Request to local ollama model
defmodule OllamaClient do
# 20 mins
@timeout 1_200_000
# "http://localhost:11434/api/generate"
@url "http://localhost:11434/v1/chat/completions"
@sys_prompt """
As a highly experienced software developer with a degree in computer science and a background in competitive programming, strive to:
1. Write efficient and concise solutions.
2. Prioritize clarity and readability in your code.
3. Include only necessary comments, focusing on explaining any tricky or complex parts of the code.
4. Ensure your code is well-organized and easy to follow.
5. Use appropriate data structures and algorithms for the problem at hand.
6. Follow best practices and coding standards for the language you are using.
By following these guidelines, you can write high-quality code that is both efficient and maintainable.
"""
@system_prompt """
You are #{Constants.lang()} programmer that writes simple, concise code and no comments or explanation.
Write #{Constants.lang()} program that it reads its input from a file "input.txt" and solve the following task.
The answer MUST BE printed to the stdout.
TASK:
"""
def call(model, task, system_prompt \\ @system_prompt) do
prompt = "#{system_prompt}#{task}"
payload =
Jason.encode!(%{
"model" => model,
"messages" => [
# %{
# "role" => "system",
# "content" => @sys_prompt
# },
%{
"role" => "user",
"content" => prompt
}
],
"stream" => false,
"temperature" => 0.5
})
case Req.post(@url, body: payload, receive_timeout: @timeout) do
{:ok, resp} ->
body = resp.body
choice = hd(body["choices"] || [%{}])
choice["message"]["content"]
error ->
IO.puts("#{__MODULE__}: #{inspect(error)}")
""
end
end
end
# resp = OllamaClient.call("neural-chat", "Write me a Nim program that reads a number from a file input.txt and multiply it by itself and prints the answer to the stdout.", "")
Code evaluator
defmodule EvaluateCode do
def call(code, file_content) do
try do
File.write!("test#{Constants.file_ext()}", code |> IO.inspect(label: "code"))
File.write!("input.txt", file_content)
# System.cmd(Constants.compiler(), ["test#{Constants.file_ext()}", "-include-runtime", "-d", "test.jar"], stderr_to_stdout: true)
# System.cmd("/bin/chmod", ["755", "test#{Constants.file_ext()}"])
# System.cmd(Constants.compiler(), ["-framework", "Foundation", "-o", "test", "test#{Constants.file_ext()}"], stderr_to_stdout: true)
# System.cmd(Constants.compiler(), ["test#{Constants.file_ext()}"], stderr_to_stdout: true)
# System.cmd(Constants.compiler(), ["test#{Constants.file_ext()}", "-o", "test"],
# stderr_to_stdout: true
# )
# System.cmd(Constants.compiler(), ["-framework", "Foundation", "test#{Constants.file_ext()}", "-o", "test"]) |> IO.inspect(label: 1)
# System.cmd("chmod", ["+x", "test#{Constants.file_ext()}"]) |> IO.inspect(label: 2)
{:ok, {result, exit_status}} =
RunWithTimeout.call(
fn ->
# System.cmd("#{File.cwd!()}/test", [], stderr_to_stdout: true)
# args = ["run", "--rm", "--platform", "linux/amd64", "-i", "-v", "#{File.cwd!()}/test.d:/test.d", "-v", "#{File.cwd!()}/input.txt:/input.txt", "dlang2/dmd-ubuntu", "/bin/bash", "-c", "dmd test.d && ./test"]
# args = ["test#{Constants.file_ext()}"]
# args = ["test#{Constants.file_ext()}"]
# args = ["-jar", "test.jar"]
# args = ["-q", "-f", "test#{Constants.file_ext()}", "-t", "halt"]
# args = [
# "-c",
# "/usr/local/bin/docker run --rm -v /Users/isavita:/app dlang2/dmd-ubuntu:latest bash -c \"cd /app && dmd -run test.d\""
# ]
args = ["-M", "test#{Constants.file_ext()}"]
# current_path = System.get_env("PATH")
# node_bin_path = "/Users/isavita/.nvm/versions/node/v19.8.1/bin"
# new_path = "#{node_bin_path}:#{current_path}"
# System.cmd(Constants.runtime(), args,
# stderr_to_stdout: true
# # env: [{"PATH", new_path}, {"LC_ALL", "en_US.UTF-8"}, {"LANG", "en_US.UTF-8"}]
# )
node_path = "/Users/isavita/.nvm/versions/node/v19.8.1/bin:#{System.get_env("PATH")}"
System.cmd(Constants.runtime(), args,
stderr_to_stdout: true,
env: [
{"PATH", node_path},
{"LC_ALL", "en_US.UTF-8"},
{"LANG", "en_US.UTF-8"},
{"GUILE_AUTO_COMPILE", "0"}
]
)
|> IO.inspect(label: "run")
end,
60_000
)
|> IO.inspect(label: "here")
if exit_status == 0 do
{:ok, inspect(result)}
else
{:error, inspect(result)}
end
rescue
exception ->
{:error, Exception.message(exception)}
end
end
end
Add mutiple time FunctionRunner
defmodule FunctionRunner do
def run_max_times(function, max_attempts) do
do_run_max_times(function, max_attempts, 1)
end
defp do_run_max_times(function, max_attempts, attempt) when attempt <= max_attempts do
case function.() do
:ok -> :ok
_ -> do_run_max_times(function, max_attempts, attempt + 1)
end
end
defp do_run_max_times(_function, _max_attempts, _attempt), do: :error
end
defmodule CodeResponseSanitizer do
def call(input) when is_binary(input) do
case Regex.scan(~r/```(?:#{Constants.lang()}|objective-c|objc)?(.*?)```/sim, input) do
[[_full_match, code] | _tail] -> code
_ -> input |> IO.inspect(label: "NO MATCH")
end
end
end
defmodule TaskSolver do
@examples """
Here are a few examples of #{Constants.lang()} language solutions for some coding challenge problems:
1. Solution:
```#{Constants.lang()}
import std.stdio;
import std.file;
import std.conv;
struct Ship {
int x, y;
int facing;
}
void main() {
auto file = File("input.txt", "r");
auto ship = Ship(0, 0, 0);
foreach(line; file.byLine()) {
auto action = line[0];
auto value = to!int(line[1..$]);
processInstruction(ship, action, value);
}
auto manhattanDistance = abs(ship.x) + abs(ship.y);
writeln(manhattanDistance);
}
void processInstruction(ref Ship ship, char action, int value) {
switch (action) {
case 'N':
ship.y += value;
break;
case 'S':
ship.y -= value;
break;
case 'E':
ship.x += value;
break;
case 'W':
ship.x -= value;
break;
case 'L':
ship.facing = (ship.facing - value + 360) % 360;
break;
case 'R':
ship.facing = (ship.facing + value) % 360;
break;
case 'F':
switch (ship.facing) {
case 0:
ship.x += value;
break;
case 90:
ship.y -= value;
break;
case 180:
ship.x -= value;
break;
case 270:
ship.y += value;
break;
default:
assert(0, "Invalid facing direction");
}
break;
default:
assert(0, "Invalid action");
}
}
int abs(int x) {
return x < 0 ? -x : x;
}
```
2. Solution:
```#{Constants.lang()}
import std.file;
import std.stdio;
void main()
{
string input = cast(string) read("input.txt");
int x = 0;
int y = 0;
int[][] visited;
visited ~= [0, 0];
foreach (char direction; input)
{
if (direction == '^')
y++;
else if (direction == 'v')
y--;
else if (direction == '>')
x++;
else if (direction == '<')
x--;
bool found = false;
foreach (coord; visited)
{
if (coord[0] == x && coord[1] == y)
{
found = true;
break;
}
}
if (!found)
visited ~= [x, y];
}
writeln(visited.length);
}
```
3. Solution:
```#{Constants.lang()}
import std.stdio;
import std.file;
import std.conv;
const favoriteNumber = 1362;
struct Point {
int x, y;
}
bool isWall(int x, int y) {
int num = x*x + 3*x + 2*x*y + y + y*y + favoriteNumber;
int bits;
while (num > 0) {
if (num % 2 == 1) bits++;
num /= 2;
}
return bits % 2 != 0;
}
int bfsMaxSteps(Point start, int maxSteps) {
bool[Point] visited;
Point[] queue;
queue ~= start;
visited[start] = true;
ulong steps; // Change type to ulong
while (queue.length > 0 && steps < cast(ulong)maxSteps) {
ulong size = cast(ulong)queue.length; // Cast to ulong
for (ulong i = 0; i < size; i++) {
Point point = queue[i];
foreach (Point delta; [Point(1, 0), Point(-1, 0), Point(0, 1), Point(0, -1)]) {
Point next = Point(point.x + delta.x, point.y + delta.y);
if (next.x >= 0 && next.y >= 0 && !isWall(next.x, next.y) && !(next in visited)) {
visited[next] = true;
queue ~= next;
}
}
}
queue = queue[size..$];
steps++;
}
return cast(int)visited.length; // Cast to int
}
void main() {
Point start = Point(1, 1);
int reachableLocations = bfsMaxSteps(start, 50);
writeln(reachableLocations);
}
```
"""
def call(day, model \\ "gpt-3.5-turbo-0125") do
task = Map.fetch!(day, "task")
input = Map.fetch!(day, "input")
answer = Map.fetch!(day, "answer")
# solution = Map.fetch!(day, "solution")
system_prompt = """
Write an #{Constants.lang()} program that reads input from a file called input.txt and prints the output to standard output.
Focus on writing clean, efficient code that demonstrates your programming skills by concisely solving the challenge.
Coding challenge:
"""
# system_prompt = """
# You are an expert programmer that writes simple, concise code and no comments or explanation.
# Given this golang program write a #{Constants.lang()} version that it reads its input from a file "input.txt" and solve the following task.
# The program should print the answer.
# #{@examples}
# Golang Solution:
# """
path =
~s|#{System.get_env("HOME")}/code/advent_generated/#{day["name"]}#{Constants.file_ext()}|
code = generate_solution(model, task, system_prompt)
with {:ok, result} <- EvaluateCode.call(code, input),
true <- valid_solution?(result, answer) do
File.write!(path, code)
{:ok, path}
else
{:error, err} ->
handle_error(model, code, err, input, answer, path)
error ->
error
end
end
defp generate_solution(model, propmt, system_prompt) do
# ft:gpt-3.5-turbo-0125:personal:erlang-0303:8yge7H3W
# "ft:gpt-3.5-turbo-0125:personal:elixir:90xUsEr3"
# "claude-3-haiku-20240307"
# OllamaClient.call(model, propmt, system_prompt)
# AnthropicClient.call(model, propmt, system_prompt)
# (MistralClient.call(model, propmt, system_prompt) ||
# (GroqClient.call(model, propmt, system_prompt) ||
(OpenAIClient.call(model, propmt, system_prompt) ||
"")
|> IO.inspect(label: "before parse")
|> CodeResponseSanitizer.call()
|> String.trim()
end
defp handle_error(model, code, err, input, answer, path) do
prompt =
"""
Your previous solution produced following error:
```#{Constants.lang()}
#{code}
```
Produce following error:
#{err}
Give me full fixed solution without missing any code.
"""
|> IO.inspect(label: "NEW PROMT")
system_prompt = """
You are an expert in #{Constants.lang()}, skilled at analyzing provided code alongside its error messages.
Examine the issues and return a fully working code solution that addresses and resolves all identified problems.
"""
code = generate_solution(model, prompt, system_prompt) |> IO.inspect(label: "NEXT TRY")
with {:ok, result} <- EvaluateCode.call(code, input) |> IO.inspect(label: "NEXT call"),
true <- valid_solution?(result, answer) |> IO.inspect(label: "NEXT valid check") do
File.write!(path, code) |> IO.inspect(label: "it uses this")
{:ok, path}
else
error -> {:error, error}
end
end
defp valid_solution?(result, answer) do
# day8_part2_2016
# day10_part1_2018
# day1_part1_2019
# day8_part2_2019
# day11_part2_2019
# day13_part2_2021
# day10_part2_2022
String.contains?(result, answer) ||
String.contains?(result, [
".##..####.###..#..#.###..####.###....##.###...###.",
" ## #### ### # # ### #### ### ## ### ### "
]) ||
String.contains?(result, [
"#....#..#....#.....###..######....##....#....#....##....######",
"# # # # ### ###### ## # # ## ######"
]) ||
String.contains?(result, ["3.465154e+06", "3.465154e+6"]) ||
String.contains?(result, [
"####.###..####.#..#.###..\n#....#..#....#.#..#.#..#.",
"#### ### #### # # ### \n# # # # # # # # "
]) ||
String.contains?(result, [
".#....###....##.#..#.####.#..#.#....#..#.\n",
" # ### ## # # #### # # # # # \n",
" █ ███ ██ █ █ ████ █ █ █ █ █ \n"
]) ||
String.contains?(result, [
"#..#.#..#.#..#.#..#.#..#.#..#.#..#....#",
"# # # # # # # # # # # # # # #"
]) ||
String.contains?(result, [
"###..###..###...##..###...##...##..####.",
"### ### ### ## ### ## ## #### "
])
end
end
len = length(dataset_202X)
IO.puts(len)
Enum.reduce(dataset_202X, 1, fn day, count ->
FunctionRunner.run_max_times(
# growthwtf/hermes2pro7b, "claude-3-haiku-20240307"
fn ->
TaskSolver.call(
day,
"ft:gpt-4o-2024-08-06:personal:code:9zTedtDh" || "gpt-4o-mini" || "mistral-large-2407" ||
"llama-3.1-70b-versatile" ||
"claude-3-opus-20240229" ||
"gpt-4-turbo-2024-04-09"
)
end,
1
)
:io.format("~.1f%\n", [count / len * 100])
Process.sleep(0_000)
count + 1
end)
