Amazon Nova

Mix.install([
  {:aws, "~> 1.0"},
  {:hackney, "~> 1.20"},
  {:req, "~> 0.5"},
  {:kino, "~> 0.14"}
])

クライアントの準備

access_key_id_input = Kino.Input.password("ACCESS_KEY_ID")
secret_access_key_input = Kino.Input.password("SECRET_ACCESS_KEY")
region_input = Kino.Input.text("REGION")

[
  access_key_id_input,
  secret_access_key_input,
  region_input
]
|> Kino.Layout.grid(columns: 3)

client =
  AWS.Client.create(
    Kino.Input.read(access_key_id_input),
    Kino.Input.read(secret_access_key_input),
    Kino.Input.read(region_input)
  )

基盤モデル一覧

models =
  client
  |> AWS.Bedrock.list_foundation_models()
  |> elem(1)
  |> Map.get("modelSummaries")
  |> Enum.filter(fn model ->
    String.starts_with?(model["modelName"], "Nova") and String.ends_with?(model["modelId"], ":0")
  end)

keys =
  [
    "providerName",
    "modelName",
    "modelId",
    "inputModalities",
    "outputModalities"
  ]

Kino.DataTable.new(models, keys: keys)

Nova Micro

model_id = "amazon.nova-micro-v1:0"

code_generation_input = """
配列から指定数の要素を選択する組み合わせを全て生成するコードを Elixir で生成してください
コメントを日本語で適切につけてください
"""

results =
  client
  |> AWS.BedrockRuntime.invoke_model(
    model_id,
    %{
      "accept" => "application/json",
      "contentType" => "application/json",
      "messages" => [
        %{
          "role" => "user",
          "content" => [
            %{
              "text" => code_generation_input
            }
          ]
        }
      ]
    }
  )
  |> elem(1)
  |> Map.get("output")

results
|> Map.get("message")
|> Map.get("content")
|> hd()
|> Map.get("text")
|> Kino.Markdown.new()

invoke_nova_text = fn input, model_id ->
  {:ok, res, _} =
    client
    |> AWS.BedrockRuntime.invoke_model(
      model_id,
      %{
        "accept" => "application/json",
        "contentType" => "application/json",
        "messages" => [
          %{
            "role" => "user",
            "content" => [%{"text" => input}]
          }
        ]
      },
      recv_timeout: 60_000
    )

  res
  |> Map.get("output")
  |> Map.get("message")
  |> Map.get("content")
  |> hd()
  |> Map.get("text")
end

japanese_input = """
大分県について、以下の項目を教えてください
- 有名な戦国武将
- 有名な史跡
- 有名な郷土料理
- ご当地キャラクター
- 特徴的な方言
"""

japanese_input
|> invoke_nova_text.("amazon.nova-micro-v1:0")
|> Kino.Markdown.new()

Nova Lite

code_generation_input
|> invoke_nova_text.("amazon.nova-lite-v1:0")
|> Kino.Markdown.new()

japanese_input
|> invoke_nova_text.("amazon.nova-lite-v1:0")
|> Kino.Markdown.new()

invoke_nova_image = fn input_text, image, model_id ->
  {:ok, res, _} =
    client
    |> AWS.BedrockRuntime.invoke_model(
      model_id,
      %{
        "accept" => "application/json",
        "contentType" => "application/json",
        "messages" => [
          %{
            "role" => "user",
            "content" => [
              %{
                "image" => %{
                  "format" => "jpeg",
                  "source" => %{
                    "bytes" => Base.encode64(image)
                  }
                }
              },
              %{
                "text" => input_text
              }
            ]
          }
        ]
      },
      recv_timeout: 60_000
    )

  res
  |> Map.get("output")
  |> Map.get("message")
  |> Map.get("content")
  |> hd()
  |> Map.get("text")
end

image_url = "https://qiita-user-contents.imgix.net/https%3A%2F%2Fqiita-image-store.s3.ap-northeast-1.amazonaws.com%2F0%2F1485835%2Fff6c4171-c83b-dc5b-afb3-fd94bc27498b.png?ixlib=rb-4.0.0&auto=format&gif-q=60&q=75&s=8cf5ad145f3859bb9901ab29b4f8efbf"

image =
  image_url
  |> Req.get!()
  |> Map.get(:body)

"画像に写っているものを詳細に教えてください"
|> invoke_nova_image.(image, "amazon.nova-lite-v1:0")
|> Kino.Markdown.new()

video_input = Kino.Input.file("VIDEO")

video =
  video_input
  |> Kino.Input.read()
  |> Map.get(:file_ref)
  |> Kino.Input.file_path()
  |> File.read!()

Kino.Video.new(video, :mp4)

invoke_nova_video = fn input_text, video, model_id ->
  {:ok, res, _} =
    client
    |> AWS.BedrockRuntime.invoke_model(
      model_id,
      %{
        "accept" => "application/json",
        "contentType" => "application/json",
        "messages" => [
          %{
            "role" => "user",
            "content" => [
              %{
                "video" => %{
                  "format" => "mp4",
                  "source" => %{
                    "bytes" => Base.encode64(video)
                  }
                }
              },
              %{
                "text" => input_text
              }
            ]
          }
        ]
      },
      recv_timeout: 600_000
    )

  res
  |> Map.get("output")
  |> Map.get("message")
  |> Map.get("content")
  |> hd()
  |> Map.get("text")
end

"動画に写っているものを説明してください"
|> invoke_nova_video.(video, "amazon.nova-lite-v1:0")
|> Kino.Markdown.new()

pdf_url = "https://www.city.chuo.lg.jp/documents/4167/chirashi.pdf"

pdf =
  pdf_url
  |> Req.get!()
  |> Map.get(:body)

invoke_nova_doc = fn input_text, doc, model_id ->
  {:ok, res, _} =
    client
    |> AWS.BedrockRuntime.invoke_model(
      model_id,
      %{
        "accept" => "application/json",
        "contentType" => "application/json",
        "messages" => [
          %{
            "role" => "user",
            "content" => [
              %{
                "document" => %{
                  "format" => "pdf",
                  "name" => "計算書類",
                  "source" => %{
                    "bytes" => Base.encode64(doc)
                  }
                }
              },
              %{
                "text" => input_text
              }
            ]
          }
        ]
      },
      recv_timeout: 600_000
    )

  res
  |> Map.get("output")
  |> Map.get("message")
  |> Map.get("content")
  |> hd()
  |> Map.get("text")
end

"何をきっかけとした取り組みですか"
|> invoke_nova_doc.(pdf, "amazon.nova-lite-v1:0")
|> Kino.Markdown.new()

Nova Pro

code_generation_input
|> invoke_nova_text.("amazon.nova-pro-v1:0")
|> Kino.Markdown.new()

japanese_input
|> invoke_nova_text.("amazon.nova-pro-v1:0")
|> Kino.Markdown.new()

"画像に写っているものを詳細に教えてください"
|> invoke_nova_image.(image, "amazon.nova-pro-v1:0")
|> Kino.Markdown.new()

"動画に写っているものを説明してください"
|> invoke_nova_video.(video, "amazon.nova-pro-v1:0")
|> Kino.Markdown.new()

"何をきっかけとした取り組みですか"
|> invoke_nova_doc.(pdf, "amazon.nova-pro-v1:0")
|> Kino.Markdown.new()

Nova Canvas

generate_image = fn input_text ->
  {:ok, res, _} =
    client
    |> AWS.BedrockRuntime.invoke_model(
      "amazon.nova-canvas-v1:0",
      %{
        "accept" => "application/json",
        "contentType" => "application/json",
        "taskType" => "TEXT_IMAGE",
        "textToImageParams" => %{
          "text" => input_text
        }
      },
      recv_timeout: 600_000
    )

  res
  |> Map.get("images")
  |> hd()
  |> Base.decode64!()
end

generate_image.("ラーメンを食べる人")

remove_image_background = fn image ->
  {:ok, res, _} =
    client
    |> AWS.BedrockRuntime.invoke_model(
      "amazon.nova-canvas-v1:0",
      %{
        "accept" => "application/json",
        "contentType" => "application/json",
        "taskType" => "BACKGROUND_REMOVAL",
        "backgroundRemovalParams" => %{
          "image" => Base.encode64(image)
        }
      },
      recv_timeout: 600_000
    )

  res
  |> Map.get("images")
  |> hd()
  |> Base.decode64!()
end

remove_image_background.(image)

edit_image = fn input_text, image ->
  {:ok, res, _} =
    client
    |> AWS.BedrockRuntime.invoke_model(
      "amazon.nova-canvas-v1:0",
      %{
        "accept" => "application/json",
        "contentType" => "application/json",
        "taskType" => "TEXT_IMAGE",
        "textToImageParams" => %{
          "conditionImage" => Base.encode64(image),
          "text" => input_text
        }
      },
      recv_timeout: 600_000
    )

  res
  |> Map.get("images")
  |> hd()
  |> Base.decode64!()
end

edit_image.("和風にして", image)

Nova Reel

generate_video = fn input_text, s3_uri ->
  {:ok, res, _} =
    client
    |> AWS.BedrockRuntime.start_async_invoke(%{
      "modelId" => "amazon.nova-reel-v1:0",
      "modelInput" => %{
        "taskType" => "TEXT_VIDEO",
        "textToVideoParams" => %{
          "text" => input_text
        },
        "videoGenerationConfig" => %{
          "durationSeconds" => 6,
          "fps" => 24,
          "dimension" => "1280x720", 
          "seed" => 12
        }
      },
      "outputDataConfig" => %{
        "s3OutputDataConfig" => %{
          "s3Uri" => s3_uri
        }
      }
    })

  res
end

s3_uri_input = Kino.Input.text("S3_URI")

s3_uri = Kino.Input.read(s3_uri_input)

generate_video.("クリスマスにサンタがダンスする可愛いアニメーション", s3_uri)

invokes =
  client
  |> AWS.BedrockRuntime.list_async_invokes()
  |> elem(1)
  |> Map.get("asyncInvokeSummaries")

video_url =
  invokes
  |> hd()
  |> Map.get("outputDataConfig")
  |> Map.get("s3OutputDataConfig")
  |> Map.get("s3Uri")
  |> Kernel.<>("/output.mp4")

bucket = video_url |> String.split("/") |> Enum.at(2)
key = video_url |> String.split("/") |> Enum.slice(3..-1//1) |> Enum.join("/")

{:ok, %{"Body" => generated_video}, _} = AWS.S3.get_object(client, bucket, key)

Kino.Video.new(generated_video, :mp4)

animate_image = fn input_text, image, s3_uri ->
  {:ok, res, _} =
    client
    |> AWS.BedrockRuntime.start_async_invoke(%{
      "modelId" => "amazon.nova-reel-v1:0",
      "modelInput" => %{
        "taskType" => "TEXT_VIDEO",
        "textToVideoParams" => %{
          "text" => input_text,
          "images" => [
            %{
              "format" => "jpeg",
              "source" => %{
                "bytes" => Base.encode64(image)
              }
            }
          ]
        },
        "videoGenerationConfig" => %{
          "durationSeconds" => 6,
          "fps" => 24,
          "dimension" => "1280x720", 
          "seed" => 12
        }
      },
      "outputDataConfig" => %{
        "s3OutputDataConfig" => %{
          "s3Uri" => s3_uri
        }
      }
    })

  res
end

start_image_input = Kino.Input.image("IMAGE", format: :jpeg)

start_image =
  start_image_input
  |> Kino.Input.read()
  |> Map.get(:file_ref)
  |> Kino.Input.file_path()
  |> File.read!()

animate_image.("ダンスさせてください", start_image, s3_uri)

invokes =
  client
  |> AWS.BedrockRuntime.list_async_invokes()
  |> elem(1)
  |> Map.get("asyncInvokeSummaries")

video_url =
  invokes
  |> hd()
  |> Map.get("outputDataConfig")
  |> Map.get("s3OutputDataConfig")
  |> Map.get("s3Uri")
  |> Kernel.<>("/output.mp4")

bucket = video_url |> String.split("/") |> Enum.at(2)
key = video_url |> String.split("/") |> Enum.slice(3..-1//1) |> Enum.join("/")

{:ok, %{"Body" => generated_video}, _} = AWS.S3.get_object(client, bucket, key)

Kino.Layout.grid([
  start_image,
  Kino.Video.new(generated_video, :mp4)
])