AWS NPM packages downloads

Mix.install([
  {:finch, "~> 0.16.0"},
  {:jsonrs, "~> 0.3.1"},
  {:kino_vega_lite, "~> 0.1.9"}
])

Supervisor.start_link(
  [
    {Finch, name: MyFinch},
    {Task.Supervisor, name: MyTaskSup}
  ],
  strategy: :one_for_one,
  name: MySup
)

Section

defmodule Npm do
  require Logger

  @registry "https://api.npmjs.org"
  @search_point "https://api.npms.io/v2/search"
  @starting "2022-01-01"
  @ending "2023-01-01"
  @search "@aws-sdk/client"

  def downloaded(packagename, start, ending) do
    path =
      @registry <>
        "/downloads/point/" <> "#{start}" <> ":" <> "#{ending}" <> "/" <> "#{packagename}"

    with {:ok, %{body: result}} <-
           Finch.build(:get, path) |> Finch.request(MyFinch),
         {:ok, response} <- Jsonrs.decode(to_string(result)) do
      response
    else
      {:error, reason} ->
        {:error, reason}
    end
  end

  def find(save? \\ false, string \\ @search, starting \\ @starting, ending \\ @ending) do
    check_response = fn
      {:ok, response} -> response
      {:error, reason} -> reason
    end

    save_to_file = fn list ->
      Task.Supervisor.async_nolink(MyTaskSup, fn ->
        case Jsonrs.encode(list, lean: true, pretty: true) do
          {:ok, result} -> File.write!("../aws-npm-packages.json", result)
          {:error, reason} -> {:error, reason}
        end
      end)
    end

    next = fn {data, page} ->
      {response, total} = search(string, 25 * page)

      case page * 25 >= total do
        true ->
          {:halt, data}

        false ->
          {response, {data, page + 1}}
      end
    end

    cross_data = fn
      nil ->
        {:error, :empty}

      %{"downloads" => d, "package" => name} ->
        Map.put(%{}, name, d)
    end

    try do
      {:ok,
       Stream.resource(
         fn -> {[], 0} end,
         &amp;next.(&amp;1),
         fn _ -> nil end
       )
       |> Task.async_stream(&amp;downloaded(&amp;1, starting, ending), timeout: 10_000)
       |> Stream.map(&amp;check_response.(&amp;1))
       |> Enum.sort_by(&amp;Map.get(&amp;1, "downloads"), :desc)
       |> tap(fn data -> if save?, do: save_to_file.(data) end)
       |> Enum.map(&amp;cross_data.(&amp;1))}
    rescue
      e ->
        Logger.warn(e)
        {:error, :emtpy}
        #     Process.sleep(1_000)
        #     find(string, starting, ending)
    end
  end

  def search(string, from \\ 0) do
    url =
      @search_point
      |> URI.new!()
      |> URI.append_query(URI.encode_query(%{q: string, size: 25, from: from}))
      |> URI.to_string()

    with {:ok, %{body: body}} <-
           Finch.build(:get, url)
           |> Finch.request(MyFinch),
         {:ok, %{"results" => results, "total" => total}} <- Jsonrs.decode(body) do
      {
        Stream.filter(results, fn package ->
          Map.has_key?(package, "flags") === false &amp;&amp;
            get_in(package, ["package", "name"]) |> String.contains?(string)
        end)
        |> Stream.map(&amp;get_in(&amp;1, ["package", "name"])),
        total
      }
    else
      {:error, reason} ->
        {:error, reason}
    end
  end
end

data = Npm.find(false, "@aws-sdk/client", "2022-01-01", "2023-01-01")

VegaLite.new(width: 600, height: 400)
|> VegaLite.data_from_values(data, only: ["downloads"])
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "downloads", type: :quantitative)
|> VegaLite.encode_field(:y, "downloads", type: :quantitative)

data = Enum.map(data, fn %{"downloads" => d, "package" => name} -> Map.put(%{}, name, d) end)