GPT-4 vision
Mix.install([
:req,
:json
])
api_key = System.fetch_env!("LB_OPENAI_API_KEY")
api_url = "https://api.openai.com/v1/chat/completions"
Load the image
image_path = "#{System.user_home!()}/git/green-bite/tmp/photo4.jpeg"
image = File.read!(image_path)
image_base64 = Base.encode64(image)
Prepare Prompt for format response with examples
user_msg = """
Identify the food in the image and if it's a dish all of its ingredients and their quantities in GRAMS.
Focus on ingredients that have CO2e values.
I will use this data to calculate the CO2e food print of the food.
Whenever you are not sure or there is not enough information in the image for the ingredients use your broder knowledge to give some estimates for the ingredients and their quantities.
RESPONSE FORMAT:
The response have to be JSON formatted surrounded with the following markup ```json```.
The JSON should have the following keys
- "food_name" - it is the name of the dish or the item if any found in the image; if no dish found in the image set it to string "none".
- "ingredients" - ARRAY of all important ingredients for caclulating the CO2e emission of the dish.
Each ingredient have to include the following keys
- "name" - the name of the ingredient.
- "grams" - quantity ALWAYS in grams.
If there is no entry
RESPONSE EXAMPLES:
1. Example of Black Bean Soup
```json
{
"food_name": "Black Bean Soup",
"ingredients": [
{"name": "black beans","grams": 250},
{"name": "carrot","grams": 80},
{"name": "yellow onion","grams": 100},
{"name": "garlic cloves", "grams": 4},
{"name": "broth", "grams": 14},
{"name": "oregano", "grams": 3},
{"name": "olive oil", "grams": 6},
{"name": "coriander", "grams": 2}
]
}
```
2. Example of single item
```json
{
"food_name": "apple",
"ingredients": [
{"name": "apple","grams": 200}
]
}
```
3. Example of no food image
```json
{
"food_name": "none",
"ingredients": []
}
```
DO NOT ADD ANYTHING ELSE to your response
"""
Prepare the payload
headers = %{
"Content-Type" => "application/json",
"Authorization" => "Bearer #{api_key}"
}
payload = %{
"model" => "gpt-4-vision-preview",
"messages" => [
%{
"role" => "user",
"content" => [
%{
"type" => "text",
"text" => user_msg
},
%{
"type" => "image_url",
"image_url" => %{
"url" => "data:image/jpeg;base64,#{image_base64}"
}
}
]
}
],
"max_tokens" => 4096
}
Make request to OpenAI Vision API
body = Req.post!(api_url, json: payload, headers: headers, receive_timeout: 60000)
Response from Photo1
resp = %Req.Response{
status: 200,
headers: %{
"alt-svc" => ["h3=\":443\"; ma=86400"],
"cf-cache-status" => ["DYNAMIC"],
"cf-ray" => ["834f760e9b3060ef-LHR"],
"connection" => ["keep-alive"],
"content-type" => ["application/json"],
"date" => ["Wed, 13 Dec 2023 16:07:48 GMT"],
"openai-model" => ["gpt-4-1106-vision-preview"],
"openai-organization" => ["user-doj5bwaqxddwo3ojyf5mulko"],
"openai-processing-ms" => ["6241"],
"openai-version" => ["2020-10-01"],
"server" => ["cloudflare"],
"set-cookie" => [
"__cf_bm=KOXSb_oVB2kad2CgebY.dh1PDrSGe4o9nEwV78vol1U-1702483668-1-AQTdf5+3Z4+WbaDujw/dy1Ni+uGuYre/r9FMYSSdc9KRioT/hfotq/h0L37w9ThGimbSLZb5l8x6JaSCuHnEbMw=; path=/; expires=Wed, 13-Dec-23 16:37:48 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None",
"_cfuvid=xKBbZ.fzohv70cryWjedfIttxz6S6yAWru8tJGKEzwg-1702483668485-0-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None"
],
"strict-transport-security" => ["max-age=15724800; includeSubDomains"],
"transfer-encoding" => ["chunked"],
"x-ratelimit-limit-requests" => ["500"],
"x-ratelimit-limit-tokens" => ["10000"],
"x-ratelimit-remaining-requests" => ["498"],
"x-ratelimit-remaining-tokens" => ["1819"],
"x-ratelimit-reset-requests" => ["5m18.66s"],
"x-ratelimit-reset-tokens" => ["49.08s"],
"x-request-id" => ["49847a061dc47aecc9931b91c883e381"]
},
body: %{
"choices" => [
%{
"finish_details" => %{"stop" => "<|fim_suffix|>", "type" => "stop"},
"index" => 0,
"message" => %{
"content" =>
"```json\n{\n\"food_name\": \"Open Faced Liver Pate Sandwich with Pickles\",\n\"ingredients\": [\n {\"name\": \"liver pate\", \"grams\": 50},\n {\"name\": \"rye bread\", \"grams\": 70},\n {\"name\": \"pickle\", \"grams\": 30}\n]\n}\n```",
"role" => "assistant"
}
}
],
"created" => 1_702_483_666,
"id" => "chatcmpl-8VM3uKOsOVLz0zPav79tS0F1lAUSh",
"model" => "gpt-4-1106-vision-preview",
"object" => "chat.completion",
"usage" => %{"completion_tokens" => 71, "prompt_tokens" => 1207, "total_tokens" => 1278}
},
trailers: %{},
private: %{}
}
choice = List.first(resp.body["choices"] || [], %{})
content1 = choice["message"]["content"]
json_content1 =
content1
|> String.split(~r/```json\s*/)
|> List.last("")
|> String.split(~r/```/)
|> List.first("")
json_content1 |> Jason.decode()
Response photo4
%Req.Response{
status: 200,
headers: %{
"alt-svc" => ["h3=\":443\"; ma=86400"],
"cf-cache-status" => ["DYNAMIC"],
"cf-ray" => ["8351b3e4ea352dd3-MAN"],
"connection" => ["keep-alive"],
"content-type" => ["application/json"],
"date" => ["Wed, 13 Dec 2023 22:39:36 GMT"],
"openai-model" => ["gpt-4-1106-vision-preview"],
"openai-organization" => ["user-doj5bwaqxddwo3ojyf5mulko"],
"openai-processing-ms" => ["10783"],
"openai-version" => ["2020-10-01"],
"server" => ["cloudflare"],
"set-cookie" => [
"__cf_bm=iibgWzs_YoRwXvQie88btUqLrPppidoozmUjHxcHqXk-1702507176-1-ATjGfPfkdBlTXZ411phQJUH/MrJ9oN9l3ZMpJrJG5yf/U5LATL9FR8WC0dqnLqzTgz+KjorHtYAzX7B7qq+Vpqs=; path=/; expires=Wed, 13-Dec-23 23:09:36 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None",
"_cfuvid=cRD1U1Y5orH65bcQHvFzQmEYI3mOSVhv3BMWvJYP4Oo-1702507176889-0-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None"
],
"strict-transport-security" => ["max-age=15724800; includeSubDomains"],
"transfer-encoding" => ["chunked"],
"x-ratelimit-limit-requests" => ["500"],
"x-ratelimit-limit-tokens" => ["10000"],
"x-ratelimit-remaining-requests" => ["495"],
"x-ratelimit-remaining-tokens" => ["5502"],
"x-ratelimit-reset-requests" => ["13m26.021s"],
"x-ratelimit-reset-tokens" => ["26.988s"],
"x-request-id" => ["d8a24dc60ffc39698dc8d5da9da96d0c"]
},
body: %{
"choices" => [
%{
"finish_details" => %{"stop" => "<|fim_suffix|>", "type" => "stop"},
"index" => 0,
"message" => %{
"content" =>
"```json\n{\n\"food_name\": \"Vegetable Empanada\",\n\"ingredients\": [\n {\"name\": \"flour\",\"grams\": 120},\n {\"name\": \"peas\",\"grams\": 30},\n {\"name\": \"carrot\",\"grams\": 20},\n {\"name\": \"onion\",\"grams\": 10},\n {\"name\": \"bell pepper\",\"grams\": 10},\n {\"name\": \"corn\",\"grams\": 20},\n {\"name\": \"olive oil\",\"grams\": 5}\n]\n}\n```",
"role" => "assistant"
}
}
],
"created" => 1_702_507_172,
"id" => "chatcmpl-8VSB21F2d7yU0txHBEPWS1LsmFLZg",
"model" => "gpt-4-1106-vision-preview",
"object" => "chat.completion",
"usage" => %{"completion_tokens" => 114, "prompt_tokens" => 1207, "total_tokens" => 1321}
},
trailers: %{},
private: %{}
}
Add StringExtractor
defmodule StringExtractor do
def extract_content(str) do
start_pattern = "```json"
end_pattern = "```"
case String.split(str, start_pattern) do
[_before, rest] ->
case String.split(rest, end_pattern) do
[content | _] -> content
_ -> str
end
_ ->
str
end
end
end
data2 = StringExtractor.extract_content(content1)
Make request to OpenAI Assistant API
system_msg = """
You are an assistant that calculates the carbon footprint of recipes (e.g. Black Bean Soup) or single food item (e.g. Apple).
You are an expert in analyzing ingredient emission factors and quantifying CO2e values for prepared meals using Python.
Users provide a recipe name and a list of ingredients with the ingredient name and quantity in grams.
Your role is to analyze the ingredients, determine the CO2e emissions produced from farming/transporting/storing each ingredient, sum the emissions for the full recipe, and report the total carbon footprint in grams.
For example, a user could provide:
1. A recipe
Input:
{
"food_name": "Open Faced Liver Pate Sandwich with Pickles",
"ingredients": [
{"name": "liver pate", "grams": 50},
{"name": "rye bread", "grams": 70},
{"name": "pickle", "grams": 30}
]
}
Write PYTHON program with the ingredients to have the SAME names as the dataset.
Whenever the ingredient is MISSING ADD SOME ESTIMATE base on your KNOWLEDGE.
```python
emissions_data = [
{"item": "pickle", "co2e_per_gram": 1.04},
{"item": "bread", "co2e_per_gram": 1.25},
{"item": "liver", "co2e_per_gram": 4.38}
]
ingredients = [
{"name": "liver", "grams": 50},
{"name": "bread", "grams": 70},
{"name": "pickle", "grams": 30}
]
def get_emissions(ingredients, emissions_data):
emissions = 0
for ingredient in ingredients:
name = ingredient["name"]
grams = ingredient["grams"]
factor = next((item for item in emissions_data if item["item"].lower() == name.lower()), None)
if factor:
emissions += factor["co2e_per_gram"] * grams
return emissions
print(get_emissions(ingredients, emissions_data))
```
Result:
337.7
2. Single food item:
Input:
{
"food_name": "Apple",
"ingredients": [
{"name": "apple", "grams": 250}
]
}
Dataset:
{"item": "apple", "co2e_per_gram": 0.22}
Result:
55
- You would have access to a dataset with some common food emissions for different foods and would calculate total CO2e emissions for the full recipes provided base on this data.
If there is no exact data use some close food data or use your broder knowledge to make educated guess.
- ALWAYS do all calculations with PYTHON!
- DO NOT RETURN ANYTHING than the result of your calculation.
- The format of your answer should be ```json```. For example: ```json{"result":55}```
"""
Prepare Model
assistant_id = "asst_U46V8wX7fdNlSHpiqirdkAsj"
# "gpt-3.5-turbo-1106"
model = "gpt-4-1106-preview"
Create Thread
threads_api_url = "https://api.openai.com/v1/threads"
threads_headers = %{
"Authorization" => "Bearer #{api_key}",
"Content-Type" => "application/json",
"OpenAI-Beta" => "assistants=v1"
}
thread_id =
Req.post!(threads_api_url, json: %{}, headers: threads_headers, receive_timeout: 60000).body[
"id"
]
Add message to a Thread
threads_messages_url = "https://api.openai.com/v1/threads/#{thread_id}/messages"
m1 = """
{
"food_name": "Open Faced Liver Pate Sandwich with Pickles",
"ingredients": [
{"name": "liver pate", "grams": 50},
{"name": "rye bread", "grams": 70},
{"name": "pickle", "grams": 30}
]
}
Calculate the total CO2 for this recipe in grams.
"""
msg1 = %{
"role" => "user",
"content" => m1
}
thread_resp =
Req.post!(threads_messages_url, json: msg1, headers: threads_headers, receive_timeout: 60000)
Get thread
threads_url = "https://api.openai.com/v1/threads/#{thread_id}"
Req.get!(threads_url, headers: threads_headers, receive_timeout: 60000)
Get thread messages
Req.get!(threads_messages_url, headers: threads_headers, receive_timeout: 60000)
Run thread
threads_run_url = "https://api.openai.com/v1/threads/#{thread_id}/runs"
threads_headers = %{
"Authorization" => "Bearer #{api_key}",
"Content-Type" => "application/json",
"OpenAI-Beta" => "assistants=v1"
}
payload = %{
"assistant_id" => assistant_id
}
thread_run_id =
Req.post!(threads_run_url, json: payload, headers: threads_headers, receive_timeout: 60000).body[
"id"
]
Get Thread’s State
run_state = Req.get!(threads_run_url, headers: threads_headers, receive_timeout: 60000)
Explore run state
last_run_id = run_state.body["last_id"]
Req.get!("https://api.openai.com/v1/threads/#{thread_id}/runs/#{last_run_id}",
headers: threads_headers
)
Explore messages
messages_url = "https://api.openai.com/v1/threads/#{thread_id}/messages"
hd(Req.get!(messages_url, headers: threads_headers, receive_timeout: 60000).body["data"] || [%{}])
Get message
messages_url = "https://api.openai.com/v1/threads/#{thread_id}/messages"
Req.get!(messages_url, headers: threads_headers, receive_timeout: 60000)
resp = %Req.Response{
status: 200,
headers: %{
"alt-svc" => ["h3=\":443\"; ma=86400"],
"cf-cache-status" => ["DYNAMIC"],
"cf-ray" => ["834987eec833771d-LHR"],
"connection" => ["keep-alive"],
"content-type" => ["application/json"],
"date" => ["Tue, 12 Dec 2023 22:51:29 GMT"],
"openai-model" => ["gpt-4-1106-vision-preview"],
"openai-organization" => ["user-XXX"],
"openai-processing-ms" => ["9238"],
"openai-version" => ["2020-10-01"],
"server" => ["cloudflare"],
"set-cookie" => [
"__cf_bm=4.epteFewE32dLTJHSrf2brlj03Rb2VEzaAsnlCv31I-1702421489-1-AWktz8GjcgwqgayxtrPjx2TeuBA7fYlc6wNfchG8DZcSiAju6asE0SqpBFq9aF73vY9sbQx2iXcUdX9fVK9kP48=; path=/; expires=Tue, 12-Dec-23 23:21:29 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None",
"_cfuvid=1tKjG_4WmoDeJq.s2j68g37jmWbBgCaTquYajwwCWYc-1702421489724-0-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None"
],
"strict-transport-security" => ["max-age=15724800; includeSubDomains"],
"transfer-encoding" => ["chunked"],
"x-ratelimit-limit-requests" => ["500"],
"x-ratelimit-limit-tokens" => ["10000"],
"x-ratelimit-remaining-requests" => ["499"],
"x-ratelimit-remaining-tokens" => ["9693"],
"x-ratelimit-reset-requests" => ["2m52.8s"],
"x-ratelimit-reset-tokens" => ["1.842s"],
"x-request-id" => ["8cb71a55516b2631efd8102b07f546c0"]
},
body: %{
"choices" => [
%{
"finish_details" => %{"stop" => "<|fim_suffix|>", "type" => "stop"},
"index" => 0,
"message" => %{
"content" =>
"This image depicts a pixel art-style scene set in a futuristic or sci-fi environment. At the center, there is an individual sitting on a chair facing away from the viewer and working on multiple computer monitors. To the left of this person, there is a robot holding a welding mask and a wrench, suggesting it's doing some technical work. Right next to the person sits a cat gazed towards the viewer, adorned with a necklace featuring what looks like technology-themed charms.\n\nOn the right, an imposing humanoid robot is being assembled or repaired while standing in some kind of teleportation or construction beam. There is a large server rack or high-tech cabinet with active screens and glowing elements to the right of the scene.\n\nIn the upper part of the image, you can see the wording \"ADVENT OF CODE 2023\" which suggests that this image might be related to an event or challenge of the same name, likely involving coding or programming.\n\nThe background features an outer space view with stars, a rocket ship, and some planets or celestial bodies, contributing to the overall space tech theme. The scene is filled with intricate details and a vibrant color palette that emphasizes the high-tech, digital aspect of the setting.",
"role" => "assistant"
}
}
],
"created" => 1_702_421_482,
"id" => "chatcmpl-8V5swEILM01x5X7kEwyMBmKerb63Y",
"model" => "gpt-4-1106-vision-preview",
"object" => "chat.completion",
"usage" => %{"completion_tokens" => 241, "prompt_tokens" => 778, "total_tokens" => 1019}
},
trailers: %{},
private: %{}
}
choice = List.first(resp.body["choices"] || [], %{})
choice["message"]["content"]
