Generate Advent Of Code solutions with LLM
Mix.install([
:req,
:json
])
Load all advent of code from hugging face
# dataset_url = "https://huggingface.co/datasets/isavita/advent-of-code/resolve/main/train.json"
# dataset = Req.get!(dataset_url, receive_timeout: 600_000).body
dataset = File.read!("/Users/isavita/git/advent-of-code/train.json") |> Jason.decode!()
defmodule Constants do
def file_ext, do: ".fs"
def lang, do: "fsharp"
def compiler, do: "/opt/homebrew/bin/dotnet"
def runtime, do: "/opt/homebrew/bin/dotnet"
end
Constants.runtime()
program = ~S|
// HelloWorld.fs
open System
[]
let main argv =
printfn "Hello, World!"
0 // Return an integer exit code
|
File.write!("test#{Constants.file_ext()}", program)
# System.cmd(Constants.compiler(), ["test#{Constants.file_ext()}", "-o", "test"],
# stderr_to_stdout: true
# )
# System.cmd("chmod", ["+x", "test#{Constants.file_ext()}"])
# System.cmd("#{File.cwd!()}/test", [], stderr_to_stdout: true)
# "#{File.cwd!()}/test#{Constants.file_ext()}"
current_path = System.get_env("PATH")
# Specify the directory where the 'node' binary can be found (not the path to the 'coffee' binary itself)
node_bin_path = "/opt/homebrew/bin/nim"
# Append the node_bin_path to the existing PATH
new_path = "#{node_bin_path}:#{current_path}"
# System.cmd("/Users/isavita/test.sh", [])
# System.cmd(Constants.runtime(), ["script", "test#{Constants.file_ext()}"], stderr_to_stdout: true)
System.cmd(Constants.runtime(), ["fsi", "test#{Constants.file_ext()}"],
stderr_to_stdout: true,
env: [{"PATH", new_path}, {"LC_ALL", "en_US.UTF-8"}, {"LANG", "en_US.UTF-8"}]
)
solutions = File.ls!("#{File.cwd!()}/code/advent_generated/fsharp")
solved_parts = Enum.map(solutions, &String.replace(&1, Constants.file_ext(), ""))
dataset_202X =
Enum.filter(dataset, fn part ->
part["solution_lang"] == "go" && part["name"] not in solved_parts
end)
Manage Request to Mistral API
defmodule MistralClient do
# 10 mins
@timeout 600_000
@url "https://api.mistral.ai/v1/chat/completions"
@system_prompt """
You are an expert programmer that writes simple, concise code and no comments or explanation.
Write a elixir module that it reads its input from a file "input.txt" and solve the following task.
The module should have only one public function called `call` with arity 0.
The answer should be RETURN instead of printed in the stdout.
TASK:
"""
def call(model, task, system_prompt \\ @system_prompt) do
prompt = "#{system_prompt}\n#{task}"
payload =
%{
"model" => model,
"messages" => [
%{"role" => "user", "content" => prompt}
],
"temperature" => 0.7,
"stream" => false
}
|> Jason.encode!()
Req.post!(@url, body: payload, headers: headers(), receive_timeout: @timeout).body
|> Map.get("choices", [%{}])
|> hd()
|> get_in(["message", "content"])
end
defp headers do
[
{"Content-Type", "application/json"},
{"Accept", "application/json"},
{"Authorization", "Bearer " <> mistral_api_key()}
]
end
defp mistral_api_key, do: System.fetch_env!("LB_MISTRAL_API_KEY")
end
defmodule OpenAIClient do
# 12 mins
@timeout 1_200_000
@url "https://api.openai.com/v1/chat/completions"
@system_prompt ~s|You are a highly experienced programmer with a PhD in computer science participating in a coding challenge.
Write clean, efficient code without unnecessary comments, demonstrating your advanced skills by solving problems practically and concisely.
Aim to produce optimal and concise solutions, leveraging your decade of industry experience.|
def call(model, prompt, system_prompt \\ @system_prompt) do
payload =
%{
"model" => model,
"temperature" => 0.5,
"messages" => [
%{"role" => "system", "content" => system_prompt},
%{"role" => "user", "content" => "#{system_prompt}\n#{prompt}"}
]
}
|> Jason.encode!()
Req.post!(@url, body: payload, headers: headers(), receive_timeout: @timeout).body
|> Map.get("choices", [%{}])
|> hd()
|> get_in(["message", "content"])
end
defp headers do
[
{"authorization", "Bearer #{openai_api_key()}"},
{"content-type", "application/json"},
{"accept", "application/json"}
]
end
defp openai_api_key, do: System.fetch_env!("LB_OPENAI_API_KEY")
end
defmodule AnthropicClient do
# 10 mins
@timeout 600_000
@url "https://api.anthropic.com/v1/messages"
@system_prompt """
As a highly experienced software developer with a degree in computer science and a background in competitive programming, strive to:
1. Write efficient and concise solutions.
2. Prioritize clarity and readability in your code.
3. Include only necessary comments, focusing on explaining any tricky or complex parts of the code.
4. Ensure your code is well-organized and easy to follow.
5. Use appropriate data structures and algorithms for the problem at hand.
6. Follow best practices and coding standards for the language you are using.
By following these guidelines, you can write high-quality code that is both efficient and maintainable.
"""
def call(model, task, system_prompt \\ @system_prompt) do
prompt = "#{system_prompt}#{task}"
payload = %{
"model" => model,
"max_tokens" => 4096,
"messages" => [
%{"role" => "user", "content" => prompt}
],
"temperature" => 0.5,
"stream" => false
}
case Req.post(@url, json: payload, headers: headers(), receive_timeout: @timeout) do
{:ok, resp} ->
if resp.status == 200 do
body = resp.body
content = hd(body["content"] || [%{}])
content["text"]
else
IO.puts("#{__MODULE__}: #{inspect(resp.body)}")
""
end
error ->
IO.puts("#{__MODULE__}: #{inspect(error)}")
""
end
end
defp headers do
%{
"content-type" => "application/json",
"anthropic-version" => "2023-06-01",
"x-api-key" => System.fetch_env!("LB_ANTHROPIC_API_KEY")
}
end
end
defmodule GroqClient do
# 5 mins
@timeout 300_000
@url "https://api.groq.com/openai/v1/chat/completions"
@system_prompt ~s|You are a highly experienced programmer with a PhD in computer science participating in a coding challenge.
Write clean, efficient code without unnecessary comments, demonstrating your advanced skills by solving problems practically and concisely.
Aim to produce optimal and concise solutions, leveraging your decade of industry experience.|
def call(model, prompt, system_prompt \\ @system_prompt) do
payload =
%{
"messages" => [
%{
"role" => "system",
"content" => system_prompt
},
%{
"role" => "user",
"content" => prompt
}
],
"model" => model,
"temperature" => 0.7,
"stream" => false
}
|> Jason.encode!()
resp = Req.post!(@url, body: payload, headers: headers(), receive_timeout: @timeout)
if resp.status == 200 do
resp.body
|> Map.get("choices", [%{}])
|> hd()
|> get_in(["message", "content"])
else
"please try again"
end
end
defp headers do
[
{"authorization", "Bearer #{groq_api_key()}"},
{"content-type", "application/json"},
{"accept", "application/json"}
]
end
defp groq_api_key, do: System.fetch_env!("LB_GROQ_API_KEY")
end
defmodule RunWithTimeout do
def call(fun, timeout) do
task =
Task.async(fn ->
try do
# Attempt to run the function
result = fun.()
{:ok, result}
rescue
exception ->
# If an exception occurs, return an error tuple
{:error, Exception.message(exception)}
end
end)
try do
Task.await(task, timeout)
rescue
# Catches exceptions and converts them to a tuple
exception ->
{:error, Exception.message(exception)}
catch
# Catches exits such as those from Task.await timeout
:exit, _ ->
# Ensures that the task is not left running
Task.shutdown(task, :brutal_kill)
# System.cmd("pgrep", ["-f", "test#{Constants.file_ext()}"], stderr_to_stdout: true)
{output, _} =
System.cmd("pgrep", ["-f", "test"], stderr_to_stdout: true)
|> IO.inspect(label: "kill")
kill_os_process(output) |> IO.inspect(label: "kill")
{:error, "The operation timed out."}
# Catches throws, which are non-standard in Elixir but still possible
:throw, value ->
{:error, "The operation was aborted: #{inspect(value)}"}
end
end
defp kill_os_process(output) do
Enum.each(String.split(output, "\n"), fn pid ->
# Ignore empty lines
if pid != "" do
# Kill each process found
System.cmd("kill", [pid])
else
:ok
end
end)
end
end
Manage Request to local ollama model
defmodule OllamaClient do
# 20 mins
@timeout 1_200_000
# "http://localhost:11434/api/generate"
@url "http://localhost:11434/v1/chat/completions"
@sys_prompt """
As a highly experienced software developer with a degree in computer science and a background in competitive programming, strive to:
1. Write efficient and concise solutions.
2. Prioritize clarity and readability in your code.
3. Include only necessary comments, focusing on explaining any tricky or complex parts of the code.
4. Ensure your code is well-organized and easy to follow.
5. Use appropriate data structures and algorithms for the problem at hand.
6. Follow best practices and coding standards for the language you are using.
By following these guidelines, you can write high-quality code that is both efficient and maintainable.
"""
@system_prompt """
You are #{Constants.lang()} programmer that writes simple, concise code and no comments or explanation.
Write #{Constants.lang()} program that it reads its input from a file "input.txt" and solve the following task.
The answer MUST BE printed to the stdout.
TASK:
"""
def call(model, task, system_prompt \\ @system_prompt) do
prompt = "#{system_prompt}#{task}"
payload =
Jason.encode!(%{
"model" => model,
"messages" => [
# %{
# "role" => "system",
# "content" => @sys_prompt
# },
%{
"role" => "user",
"content" => prompt
}
],
"stream" => false,
"temperature" => 0.5
})
case Req.post(@url, body: payload, receive_timeout: @timeout) do
{:ok, resp} ->
body = resp.body
choice = hd(body["choices"] || [%{}])
choice["message"]["content"]
error ->
IO.puts("#{__MODULE__}: #{inspect(error)}")
""
end
end
end
# resp = OllamaClient.call("neural-chat", "Write me a Nim program that reads a number from a file input.txt and multiply it by itself and prints the answer to the stdout.", "")
Code evaluator
defmodule EvaluateCode do
def call(code, file_content) do
try do
File.write!("test#{Constants.file_ext()}", code |> IO.inspect(label: "code"))
File.write!("input.txt", file_content)
# System.cmd(Constants.compiler(), ["test#{Constants.file_ext()}", "-o", "test"],
# stderr_to_stdout: true
# )
# System.cmd(Constants.compiler(), ["-framework", "Foundation", "test#{Constants.file_ext()}", "-o", "test"]) |> IO.inspect(label: 1)
# System.cmd("chmod", ["+x", "test#{Constants.file_ext()}"]) |> IO.inspect(label: 2)
{:ok, {result, exit_status}} =
RunWithTimeout.call(
fn ->
# System.cmd("#{File.cwd!()}/test", [], stderr_to_stdout: true)
# args = ["run", "--rm", "--platform", "linux/amd64", "-i", "-v", "#{File.cwd!()}/test.d:/test.d", "-v", "#{File.cwd!()}/input.txt:/input.txt", "dlang2/dmd-ubuntu", "/bin/bash", "-c", "dmd test.d && ./test"]
args = ["-f", "test#{Constants.file_ext()}"]
# args = []
current_path = System.get_env("PATH")
node_bin_path = "/Users/isavita/.nvm/versions/node/v19.8.1/bin"
new_path = "#{node_bin_path}:#{current_path}"
System.cmd(Constants.runtime(), args,
stderr_to_stdout: true,
env: [{"PATH", new_path}, {"LC_ALL", "en_US.UTF-8"}, {"LANG", "en_US.UTF-8"}]
)
|> IO.inspect(label: "run")
end,
30_000
)
|> IO.inspect(label: "here")
if exit_status == 0 do
{:ok, inspect(result)}
else
{:error, inspect(result)}
end
rescue
exception ->
{:error, Exception.message(exception)}
end
end
end
Add mutiple time FunctionRunner
defmodule FunctionRunner do
def run_max_times(function, max_attempts) do
do_run_max_times(function, max_attempts, 1)
end
defp do_run_max_times(function, max_attempts, attempt) when attempt <= max_attempts do
case function.() do
:ok -> :ok
_ -> do_run_max_times(function, max_attempts, attempt + 1)
end
end
defp do_run_max_times(_function, _max_attempts, _attempt), do: :error
end
defmodule CodeResponseSanitizer do
def call(input) when is_binary(input) do
case Regex.scan(~r/```(?:#{Constants.lang()})?(.*?)```/sim, input) do
[[_full_match, code] | _tail] -> code
_ -> input |> IO.inspect(label: "NO MATCH")
end
end
end
defmodule TaskSolver do
@examples """
Here are a few examples of #{Constants.lang()} language solutions for some coding challenge problems:
1. Solution for "--- Day 1: Calorie Counting ---" of Part One of the challenge:
```#{Constants.lang()}
#!/bin/bash
maxCalories=0
currentCalories=0
while IFS= read -r line || [ -n "$line" ]; do
if [ -z "$line" ]; then
if [ $currentCalories -gt $maxCalories ]; then
maxCalories=$currentCalories
fi
currentCalories=0
continue
fi
currentCalories=$((currentCalories + line))
done < "input.txt"
if [ $currentCalories -gt $maxCalories ]; then
maxCalories=$currentCalories
fi
echo $maxCalories
```
2. Solution for "--- Day 4: Secure Container ---" of the Part One of the challenge:
```#{Constants.lang()}
#!/bin/bash
read -r rangeStr < input.txt
IFS='-' read -r start end <<< "$rangeStr"
count=0
for ((i=start; i<=end; i++)); do
s="$i"
hasDouble=false
prevChar=${s:0:1}
for ((j=1; j<${#s}; j++)); do
currChar=${s:j:1}
if [ "$prevChar" == "$currChar" ]; then
hasDouble=true
fi
if [ "$prevChar" \> "$currChar" ]; then
hasDouble=false
break
fi
prevChar="$currChar"
done
if $hasDouble; then
((count++))
fi
done
echo $count
```
3. Solution for "--- Day 19: A Series of Tubes ---" Part Two of the challenge:
```#{Constants.lang()}
#!/bin/bash
file="input.txt"
grid=()
x=0
y=0
while IFS= read -r line; do
grid+=("$line")
done < "$file"
for ((i=0; i<${#grid[0]}; i++)); do
if [ "${grid[0]:i:1}" = "|" ]; then
x=$i
break
fi
done
dx=0
dy=1
steps=0
while true; do
if [ $x -lt 0 ] || [ $x -ge ${#grid[0]} ] || [ $y -lt 0 ] || [ $y -ge ${#grid[@]} ]; then
break
fi
cell="${grid[$y]:$x:1}"
if [ "$cell" = " " ]; then
break
fi
((steps++))
if [ "$cell" = "+" ]; then
if [ $dx -eq 0 ]; then
if [ $x -gt 0 ] && { [ "${grid[$y]:x-1:1}" = "-" ] || [ "${grid[$y]:x-1:1}" = [A-Z] ]; }; then
dx=-1
dy=0
else
dx=1
dy=0
fi
else
if [ $y -gt 0 ] && { [ "${grid[y-1]:x:1}" = "|" ] || [ "${grid[y-1]:x:1}" = [A-Z] ]; }; then
dx=0
dy=-1
else
dx=0
dy=1
fi
fi
fi
((x+=dx))
((y+=dy))
done
echo $steps
exit 0
```
"""
def call(day, model \\ "gpt-3.5-turbo-0125") do
task = Map.fetch!(day, "task")
input = Map.fetch!(day, "input")
answer = Map.fetch!(day, "answer")
# solution = Map.fetch!(day, "solution")
system_prompt = """
Write an #{Constants.lang()} program that reads input from a file called input.txt and prints the output to standard output.
Focus on writing clean, efficient code that demonstrates your programming skills by concisely solving the challenge.
**NOTE:** DO NOT forget to add at the top of the code #!/usr/bin/awk
Coding challenge:
"""
# system_prompt = """
# You are an expert programmer that writes simple, concise code and no comments or explanation.
# Given this golang program write a #{Constants.lang()} version that it reads its input from a file "input.txt" and solve the following task.
# The program should print the answer.
# **NOTE:** Golang is faster language than #{Constants.lang()}, thus you have to try more efficient solution in python!
# Golang Solution:
# """
path =
~s|#{System.get_env("HOME")}/code/advent_generated/#{day["name"]}#{Constants.file_ext()}|
code = generate_solution(model, task, system_prompt)
with {:ok, result} <- EvaluateCode.call(code, input),
true <- valid_solution?(result, answer) do
File.write!(path, code)
{:ok, path}
else
{:error, err} ->
handle_error(model, code, err, input, answer, path)
error ->
error
end
end
defp generate_solution(model, propmt, system_prompt) do
# ft:gpt-3.5-turbo-0125:personal:erlang-0303:8yge7H3W
# "ft:gpt-3.5-turbo-0125:personal:elixir:90xUsEr3"
# "claude-3-haiku-20240307"
# MistralClient.call(model, propmt, system_prompt)
# OllamaClient.call(model, propmt, system_prompt)
# OpenAIClient.call(model, propmt, system_prompt)
# AnthropicClient.call(model, propmt, system_prompt)
(GroqClient.call(model, propmt, system_prompt) ||
"")
|> IO.inspect(label: "before parse")
|> CodeResponseSanitizer.call()
|> String.trim()
end
defp handle_error(model, code, err, input, answer, path) do
prompt =
"""
Your previous solution produced following error:
```#{Constants.lang()}
#{code}
```
Produce following error:
#{err}
Give me full fixed solution without missing any code.
"""
|> IO.inspect(label: "NEW PROMT")
system_prompt = """
You are an expert in #{Constants.lang()}, skilled at analyzing provided code alongside its error messages.
Examine the issues and return a fully working code solution that addresses and resolves all identified problems.
"""
code = generate_solution(model, prompt, system_prompt) |> IO.inspect(label: "NEXT TRY")
with {:ok, result} <- EvaluateCode.call(code, input) |> IO.inspect(label: "NEXT call"),
true <- valid_solution?(result, answer) |> IO.inspect(label: "NEXT valid check") do
File.write!(path, code) |> IO.inspect(label: "it uses this")
{:ok, path}
else
error -> {:error, error}
end
end
defp valid_solution?(result, answer) do
# day8_part2_2016
# day10_part1_2018
# day1_part1_2019
# day8_part2_2019
# day11_part2_2019
# day13_part2_2021
# day10_part2_2022
String.contains?(result, answer) ||
String.contains?(result, [
".##..####.###..#..#.###..####.###....##.###...###.",
" ## #### ### # # ### #### ### ## ### ### "
]) ||
String.contains?(result, [
"#....#..#....#.....###..######....##....#....#....##....######",
"# # # # ### ###### ## # # ## ######"
]) ||
String.contains?(result, ["3.465154e+06", "3.465154e+6"]) ||
String.contains?(result, [
"####.###..####.#..#.###..\n#....#..#....#.#..#.#..#.",
"#### ### #### # # ### \n# # # # # # # # "
]) ||
String.contains?(result, [
".#....###....##.#..#.####.#..#.#....#..#.\n",
" # ### ## # # #### # # # # # \n",
" █ ███ ██ █ █ ████ █ █ █ █ █ \n"
]) ||
String.contains?(result, [
"#..#.#..#.#..#.#..#.#..#.#..#.#..#....#",
"# # # # # # # # # # # # # # #"
]) ||
String.contains?(result, [
"###..###..###...##..###...##...##..####.",
"### ### ### ## ### ## ## #### "
])
end
end
len = length(dataset_202X)
IO.puts(len)
Enum.reduce(dataset_202X, 1, fn day, count ->
FunctionRunner.run_max_times(
# growthwtf/hermes2pro7b, "claude-3-haiku-20240307"
fn ->
TaskSolver.call(
day,
"llama3-70b-8192" || "dolphin-llama3" || "claude-3-opus-20240229" ||
"gpt-4-turbo-2024-04-09"
)
end,
1
)
:io.format("~.1f%\n", [count / len * 100])
Process.sleep(15_000)
count + 1
end)
