Dataverses

Mix.install([
  {:req, "~> 0.5.6"},
  {:explorer, "~> 0.9.2"},
  {:kino_explorer, "~> 0.1.23"},
  {:kino, "~> 0.14.1"}
  # {:kino_progress_bar, "~> 0.1.1"} # would be great but depends on kino 0.12 😞
])

Section

repository = "entrepot.recherche.data.gouv.fr"

response = Req.get!(
  "https://#{repository}/api/search?type=file", 
  params: %{q: "*"}, 
  decode_body: :json
)
response.body

total_count = response.body["data"]["total_count"]
per_page = 1000
starts = 0..total_count-1 |> Enum.filter(fn k -> rem(k, per_page) == 0 end)
Kino.Markdown.new ("""

| Attribute   | Value          |
|-------------|-----------------
| total_count | #{total_count} |
| per_page    | #{per_page}    |

""")

Enum.map(starts, fn start -> 
  response = Req.get!(
    "https://#{repository}/api/search", 
    params: %{q: "*", type: "file", start: start, per_page: per_page}, 
    decode_body: :json
  )
  IO.puts "#{start} / #{total_count}"
  response.body["data"]["items"]
end)

starts
|> Task.async_stream(
    fn start -> 
      response = Req.get!(
        "https://#{repository}/api/search", 
        params: %{q: "*", type: "file", start: start, per_page: per_page}, 
        decode_body: :json,
        receive_timeout: 100_000
      )
      IO.puts "#{start} / #{total_count}"
      response.body["data"]["items"]
    end,
    ordered: true
)
|> Enum.to_list()