Ingest Data

Mix.install([
  {:explorer, "~> 0.10.1"},
  {:kino, "~> 0.15"},
  {:tama_ex, github: "upmaru/tama-ex"}
])

Setup Client

alias TamaEx.Neural
alias TamaEx.Perception
alias TamaEx.Memory

provision = TamaEx.client(base_url: "http://localhost:4000/provision")
ingest = TamaEx.client(base_url: "http://localhost:4000/ingest")

{:ok, movie_db_space} = Neural.get_space(provision, "movie-db")

{:ok, movie_details_class} = 
  Neural.get_class(provision, movie_db_space, "movie-details")

Load dataset

alias Explorer.DataFrame
alias Explorer.Series

require Explorer.DataFrame

df =
  Kino.FS.file_path("movies.json")
  |> DataFrame.from_ndjson!()

df = DataFrame.mutate(df,
  release_date: Series.cast(release_date, :date)
)

df =
  df
  |> DataFrame.filter(year(release_date) >= 2020)
  |> DataFrame.filter(not(adult))
  |> DataFrame.filter(status == "Released")
  |> DataFrame.filter(lengths(split(overview, " ")) > 5)
  |> DataFrame.sort_by(desc: popularity)

stream =
  DataFrame.to_rows_stream(df, atom_keys: true, chunk_size: 500)

Push into Tama

movies =
  stream
  |> Enum.take(5)

movies
|> Enum.map(fn movie ->
  attrs = %{
    identifier: "#{movie.id}",
    record: movie
  }

  Memory.create_entity(ingest, movie_details_class, attrs)
end)