Amazon Textract

Mix.install([
  {:aws, "~> 0.13"},
  {:hackney, "~> 1.20"},
  {:kino, "~> 0.14"}
])

クライアントの準備

access_key_id_input = Kino.Input.password("ACCESS_KEY_ID")
secret_access_key_input = Kino.Input.password("SECRET_ACCESS_KEY")
region_input = Kino.Input.text("REGION")

[
  access_key_id_input,
  secret_access_key_input,
  region_input
]
|> Kino.Layout.grid(columns: 3)

client =
  AWS.Client.create(
    Kino.Input.read(access_key_id_input),
    Kino.Input.read(secret_access_key_input),
    Kino.Input.read(region_input)
  )

画像からの表データ読込

image_input = Kino.Input.image("IMAGE", format: :png)

image_binary =
  image_input
  |> Kino.Input.read()
  |> Map.get(:file_ref)
  |> Kino.Input.file_path()
  |> File.read!()

blocks =
  client
  |> AWS.Textract.analyze_document(%{
    "Document" => %{
      "Bytes" => Base.encode64(image_binary)
    },
    "FeatureTypes" => ["TABLES"]
  })
  |> elem(1)
  |> Map.get("Blocks")

Enum.filter(blocks, fn block -> block["BlockType"] == "TABLE" end)

Enum.filter(blocks, fn block -> block["BlockType"] == "CELL" end)

Enum.filter(blocks, fn block -> block["BlockType"] == "WORD" end)

cells =
  blocks
  |> Enum.filter(fn block -> block["BlockType"] == "CELL" end)
  |> Enum.map(fn cell ->
    # 各セルについて、子要素の Text を取得する
    words =
      case cell["Relationships"] do
        nil ->
          ""

        relationships ->
          relationships
          |> Enum.filter(fn child -> child["Type"] == "CHILD" end)
          |> Enum.map(fn child ->
            Enum.map(child["Ids"], fn child_id ->
              blocks
              |> Enum.find(%{}, fn block -> block["Id"] == child_id end)
              |> Map.get("Text", "")
            end)
          end)
          |> Enum.join(" ")
      end

    cell
    |> Map.take(["ColumnIndex", "RowIndex"])
    |> Map.put("Text", words)
  end)

Kino.DataTable.new(cells)

columns =
  cells
  |> Enum.filter(fn cell -> cell["RowIndex"] == 1 end)
  |> Enum.map(fn column ->
    Map.put(column, "Text", String.slice(column["Text"], 1..-1))
  end)

max_row_index =
  cells
  |> Enum.map(fn cell -> cell["RowIndex"] end)
  |> Enum.max()

rows =
  2..max_row_index
  |> Enum.map(fn row_index ->
    Enum.filter(cells, fn cell -> cell["RowIndex"] == row_index end)
  end)

table_data =
  rows
  |> Enum.map(fn row ->
    columns
    |> Enum.into(%{}, fn column ->
      value =
        row
        |> Enum.find(fn cell -> cell["ColumnIndex"] == column["ColumnIndex"] end)
        |> Map.get("Text")

      {column["Text"], value}
    end)
  end)

Kino.DataTable.new(table_data)

画像へのクエリ

blocks =
  client
  |> AWS.Textract.analyze_document(%{
    "Document" => %{
      "Bytes" => Base.encode64(image_binary)
    },
    "FeatureTypes" => ["QUERIES"],
    "QueriesConfig" => %{
      "Queries" => [
        %{
          "Text" => "What is the Takumi's gender"
        }
      ]
    }
  })
  |> elem(1)
  |> Map.get("Blocks")

Enum.filter(blocks, fn block -> block["BlockType"] == "QUERY_RESULT" end)