Site Scanning

Mix.install([
  {:httpoison, "~> 2.2"},
  {:jason, "~> 1.4"},
  {:kino_explorer, "~> 0.1.15"},
  {:kino_vega_lite, "~> 0.1.10"}
])

Section

# Enter your open.data.gov API Key as a Livebook Secret named "API_DATA_GOV_KEY"
# For reference on Livebook secrets - https://www.youtube.com/watch?v=laG04lxagZI
{:ok, api_key} = System.fetch_env("LB_API_DATA_GOV_KEY")

Fetch all websites

url = "https://api.gsa.gov/technology/site-scanning/v1/websites?api_key=#{api_key}&limit=100"

%{body: body} = HTTPoison.get!(url)
json = Jason.decode!(body)

# Grab the "items" key from the API's json response
%{"items" => sites} = json

Show all websites

sites

Re-encode the data as .json and write it to file

json_data = sites |> Jason.encode!()

File.write!("site-scanning-sites.json", json_data)

# Alternatively, you could just write the original body of the response to file
# File.write!("site-scanning-sites.json", body)

Show one website

sites |> Enum.at(0)

Find information about a single domain

domain = "gsa.gov"

url =
  "https://api.gsa.gov/technology/site-scanning/v1/websites/#{domain}?api_key=#{api_key}&limit=100"

%{body: body} = HTTPoison.get!(url)
json = Jason.decode!(body)

Write the response to a file

File.write!("site-scanning-single-site-response.json", body)