Powered by AppSignal & Oban Pro
Would you like to see your link here? Contact us

Simulation ingestion

simul_ingestion.livemd

Simulation ingestion

Mix.install([
  {:simul_ingestion, path: "."},
  {:kino_vega_lite, "~> 0.1.13"}
])

Section

experiment_param = [
  chunking_nb: 2,
  embedding_nb: 1,
  max_batch_per_min: 400,
  embedding_time_ms: 300,
  chunking_per_sec: 1_000
]

Kino.start_child!({SimulIngestion.Pipeline, experiment_param})
data = SimulIngestion.Dashboard.get_state()
[(simul_ingestion 0.1.0) lib/simul_ingestion/embedding_service.ex:18: SimulIngestion.EmbeddingService.init/1]
Keyword.fetch!(args, :max_batch_per_min) #=> 400

[(simul_ingestion 0.1.0) lib/simul_ingestion/embedding_service.ex:19: SimulIngestion.EmbeddingService.init/1]
Keyword.fetch!(args, :embedding_time_ms) #=> 300

incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
%{docs: {0, 0, 0}, embedding_speed: [], freq: []}
# Parallélisation maximum (100 docs * n chunks / 15 chunks/batch / 400 batchs/min)
min_min = 100 * 65 / 15 / 400
1.0833333333333333
for j <- 1..20 do
  for i <- 1..5, do: SimulIngestion.User.send_small_doc("doc_#{j}_#{i}")
   Process.sleep(2_000)
end

# for i <- 1..100, do: SimulIngestion.User.send_small_doc("doc_#{i}")
...
"Embedding chunks for book doc_5_4"
"Embedding chunks for book doc_5_4"
"Embedding chunks for book doc_5_4"
"Indexing chunks for book doc_5_5"
"Embedding chunks for book doc_5_4"
"Indexing chunks for book doc_5_5"
"Indexing chunks for book doc_5_5"
"Indexing chunks for book doc_5_5"
"Indexing chunks for book doc_5_5"
"Embedding chunks for book doc_5_3"
"Indexing chunks for book doc_5_4"
"Indexing chunks for book doc_5_4"
"Indexing chunks for book doc_5_4"
"Indexing chunks for book doc_5_4"
"Indexing chunks for book doc_5_4"
"Embedding chunks for book doc_5_3"
"Embedding chunks for book doc_5_3"
"Embedding chunks for book doc_5_3"
"Embedding chunks for book doc_5_3"
"Indexing chunks for book doc_5_3"
"Indexing chunks for book doc_5_3"
"Indexing chunks for book doc_5_3"
"Indexing chunks for book doc_5_3"
"Indexing chunks for book doc_5_3"
"Adding book doc_6_1"
"Starting chunking for book doc_6_1"
"Adding book doc_6_2"
"Starting chunking for book doc_6_2"
"Adding book doc_6_3"
"Adding book doc_6_4"
"Starting chunking for book doc_6_4"
"Adding book doc_6_5"
"Starting chunking for book doc_6_3"
"Starting chunking for book doc_6_5"
incoming demand : 1
"Embedding chunks for book doc_6_1"
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_6_3"
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_6_3"
"Embedding chunks for book doc_6_3"
"Embedding chunks for book doc_6_3"
"Embedding chunks for book doc_6_1"
"Embedding chunks for book doc_6_3"
"Indexing chunks for book doc_6_3"
"Indexing chunks for book doc_6_1"
"Indexing chunks for book doc_6_3"
"Embedding chunks for book doc_6_1"
"Indexing chunks for book doc_6_3"
"Indexing chunks for book doc_6_3"
"Embedding chunks for book doc_6_1"
"Embedding chunks for book doc_6_5"
"Indexing chunks for book doc_6_1"
"Indexing chunks for book doc_6_1"
"Embedding chunks for book doc_6_5"
"Indexing chunks for book doc_6_3"
"Indexing chunks for book doc_6_5"
"Indexing chunks for book doc_6_1"
"Embedding chunks for book doc_6_5"
"Embedding chunks for book doc_6_5"
"Embedding chunks for book doc_6_1"
"Embedding chunks for book doc_6_5"
"Indexing chunks for book doc_6_5"
"Indexing chunks for book doc_6_5"
"Indexing chunks for book doc_6_1"
"Embedding chunks for book doc_6_2"
"Embedding chunks for book doc_6_2"
"Embedding chunks for book doc_6_2"
"Indexing chunks for book doc_6_5"
"Embedding chunks for book doc_6_2"
"Indexing chunks for book doc_6_5"
"Embedding chunks for book doc_6_2"
"Indexing chunks for book doc_6_2"
"Indexing chunks for book doc_6_2"
"Indexing chunks for book doc_6_2"
"Embedding chunks for book doc_6_4"
"Indexing chunks for book doc_6_2"
"Embedding chunks for book doc_6_4"
"Indexing chunks for book doc_6_2"
"Embedding chunks for book doc_6_4"
"Embedding chunks for book doc_6_4"
"Embedding chunks for book doc_6_4"
"Indexing chunks for book doc_6_4"
"Indexing chunks for book doc_6_4"
"Indexing chunks for book doc_6_4"
"Indexing chunks for book doc_6_4"
"Indexing chunks for book doc_6_4"
"Adding book doc_7_1"
"Starting chunking for book doc_7_1"
"Adding book doc_7_2"
"Adding book doc_7_3"
"Adding book doc_7_4"
"Adding book doc_7_5"
"Starting chunking for book doc_7_2"
"Starting chunking for book doc_7_4"
"Starting chunking for book doc_7_3"
"Starting chunking for book doc_7_5"
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_7_3"
incoming demand : 1
"Embedding chunks for book doc_7_3"
"Embedding chunks for book doc_7_2"
"Embedding chunks for book doc_7_3"
"Embedding chunks for book doc_7_3"
"Embedding chunks for book doc_7_2"
"Embedding chunks for book doc_7_2"
"Indexing chunks for book doc_7_3"
"Indexing chunks for book doc_7_3"
"Indexing chunks for book doc_7_3"
"Embedding chunks for book doc_7_2"
"Indexing chunks for book doc_7_2"
"Embedding chunks for book doc_7_3"
"Indexing chunks for book doc_7_3"
"Embedding chunks for book doc_7_2"
"Indexing chunks for book doc_7_2"
"Indexing chunks for book doc_7_2"
"Embedding chunks for book doc_7_1"
"Indexing chunks for book doc_7_2"
"Indexing chunks for book doc_7_3"
"Indexing chunks for book doc_7_2"
"Embedding chunks for book doc_7_1"
"Embedding chunks for book doc_7_1"
"Embedding chunks for book doc_7_1"
"Embedding chunks for book doc_7_1"
"Indexing chunks for book doc_7_1"
"Embedding chunks for book doc_7_4"
"Indexing chunks for book doc_7_1"
"Indexing chunks for book doc_7_1"
"Embedding chunks for book doc_7_4"
"Indexing chunks for book doc_7_1"
"Indexing chunks for book doc_7_1"
"Embedding chunks for book doc_7_4"
"Embedding chunks for book doc_7_4"
"Embedding chunks for book doc_7_4"
"Indexing chunks for book doc_7_4"
"Embedding chunks for book doc_7_5"
"Embedding chunks for book doc_7_5"
"Indexing chunks for book doc_7_4"
"Indexing chunks for book doc_7_4"
"Indexing chunks for book doc_7_4"
"Indexing chunks for book doc_7_4"
"Embedding chunks for book doc_7_5"
"Embedding chunks for book doc_7_5"
"Embedding chunks for book doc_7_5"
"Indexing chunks for book doc_7_5"
"Indexing chunks for book doc_7_5"
"Indexing chunks for book doc_7_5"
"Indexing chunks for book doc_7_5"
"Indexing chunks for book doc_7_5"
"Adding book doc_8_1"
"Adding book doc_8_2"
"Starting chunking for book doc_8_1"
"Starting chunking for book doc_8_2"
"Adding book doc_8_3"
"Starting chunking for book doc_8_3"
"Adding book doc_8_4"
"Adding book doc_8_5"
"Starting chunking for book doc_8_5"
"Starting chunking for book doc_8_4"
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_8_2"
incoming demand : 1
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_8_2"
"Embedding chunks for book doc_8_2"
"Embedding chunks for book doc_8_2"
"Embedding chunks for book doc_8_2"
"Embedding chunks for book doc_8_1"
"Indexing chunks for book doc_8_2"
"Indexing chunks for book doc_8_2"
"Indexing chunks for book doc_8_2"
"Indexing chunks for book doc_8_2"
"Indexing chunks for book doc_8_2"
"Embedding chunks for book doc_8_1"
"Embedding chunks for book doc_8_1"
"Embedding chunks for book doc_8_1"
"Embedding chunks for book doc_8_1"
"Embedding chunks for book doc_8_3"
"Indexing chunks for book doc_8_1"
"Indexing chunks for book doc_8_1"
"Indexing chunks for book doc_8_1"
"Indexing chunks for book doc_8_1"
"Indexing chunks for book doc_8_1"
"Embedding chunks for book doc_8_3"
"Embedding chunks for book doc_8_3"
"Embedding chunks for book doc_8_3"
"Embedding chunks for book doc_8_3"
"Indexing chunks for book doc_8_3"
"Indexing chunks for book doc_8_3"
"Indexing chunks for book doc_8_3"
"Indexing chunks for book doc_8_3"
"Embedding chunks for book doc_8_5"
"Indexing chunks for book doc_8_3"
"Embedding chunks for book doc_8_5"
"Embedding chunks for book doc_8_5"
"Embedding chunks for book doc_8_5"
"Embedding chunks for book doc_8_5"
"Embedding chunks for book doc_8_4"
"Indexing chunks for book doc_8_5"
"Indexing chunks for book doc_8_5"
"Indexing chunks for book doc_8_5"
"Indexing chunks for book doc_8_5"
"Indexing chunks for book doc_8_5"
"Embedding chunks for book doc_8_4"
"Embedding chunks for book doc_8_4"
"Embedding chunks for book doc_8_4"
"Embedding chunks for book doc_8_4"
"Indexing chunks for book doc_8_4"
"Indexing chunks for book doc_8_4"
"Indexing chunks for book doc_8_4"
"Indexing chunks for book doc_8_4"
"Indexing chunks for book doc_8_4"
"Adding book doc_9_1"
"Starting chunking for book doc_9_1"
"Adding book doc_9_2"
"Starting chunking for book doc_9_2"
"Adding book doc_9_3"
"Adding book doc_9_4"
"Starting chunking for book doc_9_3"
"Starting chunking for book doc_9_4"
"Adding book doc_9_5"
"Starting chunking for book doc_9_5"
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_9_1"
"Embedding chunks for book doc_9_1"
"Embedding chunks for book doc_9_1"
"Embedding chunks for book doc_9_1"
"Embedding chunks for book doc_9_1"
"Embedding chunks for book doc_9_3"
"Indexing chunks for book doc_9_1"
"Indexing chunks for book doc_9_1"
"Indexing chunks for book doc_9_1"
"Indexing chunks for book doc_9_1"
"Indexing chunks for book doc_9_1"
"Embedding chunks for book doc_9_3"
"Embedding chunks for book doc_9_2"
"Embedding chunks for book doc_9_3"
"Embedding chunks for book doc_9_3"
"Indexing chunks for book doc_9_3"
"Indexing chunks for book doc_9_3"
"Indexing chunks for book doc_9_3"
"Indexing chunks for book doc_9_2"
"Indexing chunks for book doc_9_3"
"Embedding chunks for book doc_9_2"
"Embedding chunks for book doc_9_2"
"Embedding chunks for book doc_9_2"
"Embedding chunks for book doc_9_3"
"Embedding chunks for book doc_9_2"
"Indexing chunks for book doc_9_2"
"Indexing chunks for book doc_9_2"
"Indexing chunks for book doc_9_2"
"Indexing chunks for book doc_9_3"
"Embedding chunks for book doc_9_4"
"Indexing chunks for book doc_9_2"
"Embedding chunks for book doc_9_4"
"Embedding chunks for book doc_9_4"
"Embedding chunks for book doc_9_4"
"Embedding chunks for book doc_9_4"
"Embedding chunks for book doc_9_5"
"Embedding chunks for book doc_9_5"
"Indexing chunks for book doc_9_4"
"Embedding chunks for book doc_9_5"
"Embedding chunks for book doc_9_5"
"Embedding chunks for book doc_9_5"
"Indexing chunks for book doc_9_4"
"Indexing chunks for book doc_9_4"
"Indexing chunks for book doc_9_4"
"Indexing chunks for book doc_9_4"
"Indexing chunks for book doc_9_5"
"Indexing chunks for book doc_9_5"
"Indexing chunks for book doc_9_5"
"Indexing chunks for book doc_9_5"
"Indexing chunks for book doc_9_5"
"Adding book doc_10_1"
"Starting chunking for book doc_10_1"
"Adding book doc_10_2"
"Adding book doc_10_3"
"Starting chunking for book doc_10_3"
"Adding book doc_10_4"
"Starting chunking for book doc_10_4"
"Adding book doc_10_5"
"Starting chunking for book doc_10_2"
"Starting chunking for book doc_10_5"
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_10_1"
"Embedding chunks for book doc_10_1"
"Embedding chunks for book doc_10_1"
"Embedding chunks for book doc_10_1"
"Embedding chunks for book doc_10_1"
"Indexing chunks for book doc_10_1"
"Indexing chunks for book doc_10_1"
"Indexing chunks for book doc_10_1"
"Indexing chunks for book doc_10_1"
"Indexing chunks for book doc_10_1"
"Embedding chunks for book doc_10_3"
"Embedding chunks for book doc_10_3"
"Embedding chunks for book doc_10_3"
"Embedding chunks for book doc_10_3"
"Embedding chunks for book doc_10_3"
"Indexing chunks for book doc_10_3"
"Indexing chunks for book doc_10_3"
"Indexing chunks for book doc_10_3"
"Indexing chunks for book doc_10_3"
"Indexing chunks for book doc_10_3"
"Embedding chunks for book doc_10_4"
"Embedding chunks for book doc_10_4"
"Embedding chunks for book doc_10_4"
"Embedding chunks for book doc_10_4"
"Embedding chunks for book doc_10_4"
"Indexing chunks for book doc_10_4"
"Indexing chunks for book doc_10_4"
"Indexing chunks for book doc_10_4"
"Indexing chunks for book doc_10_4"
"Indexing chunks for book doc_10_4"
"Embedding chunks for book doc_10_2"
"Embedding chunks for book doc_10_2"
"Embedding chunks for book doc_10_2"
"Embedding chunks for book doc_10_2"
"Embedding chunks for book doc_10_2"
"Indexing chunks for book doc_10_2"
"Embedding chunks for book doc_10_5"
"Indexing chunks for book doc_10_2"
"Indexing chunks for book doc_10_2"
"Indexing chunks for book doc_10_2"
"Indexing chunks for book doc_10_2"
"Embedding chunks for book doc_10_5"
"Embedding chunks for book doc_10_5"
"Embedding chunks for book doc_10_5"
"Embedding chunks for book doc_10_5"
"Indexing chunks for book doc_10_5"
"Indexing chunks for book doc_10_5"
"Indexing chunks for book doc_10_5"
"Indexing chunks for book doc_10_5"
"Indexing chunks for book doc_10_5"
"Adding book doc_11_1"
"Starting chunking for book doc_11_1"
"Adding book doc_11_2"
"Starting chunking for book doc_11_2"
"Adding book doc_11_3"
"Adding book doc_11_4"
"Starting chunking for book doc_11_4"
"Adding book doc_11_5"
"Starting chunking for book doc_11_5"
"Starting chunking for book doc_11_3"
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_11_1"
"Embedding chunks for book doc_11_1"
"Embedding chunks for book doc_11_1"
"Embedding chunks for book doc_11_1"
"Embedding chunks for book doc_11_1"
"Indexing chunks for book doc_11_1"
"Indexing chunks for book doc_11_1"
"Indexing chunks for book doc_11_1"
"Indexing chunks for book doc_11_1"
"Embedding chunks for book doc_11_2"
"Embedding chunks for book doc_11_2"
"Embedding chunks for book doc_11_2"
"Indexing chunks for book doc_11_1"
"Embedding chunks for book doc_11_2"
"Embedding chunks for book doc_11_2"
"Indexing chunks for book doc_11_2"
"Indexing chunks for book doc_11_2"
"Indexing chunks for book doc_11_2"
"Indexing chunks for book doc_11_2"
"Indexing chunks for book doc_11_2"
"Embedding chunks for book doc_11_4"
"Embedding chunks for book doc_11_4"
"Embedding chunks for book doc_11_4"
"Embedding chunks for book doc_11_4"
"Embedding chunks for book doc_11_4"
"Indexing chunks for book doc_11_4"
"Indexing chunks for book doc_11_4"
"Indexing chunks for book doc_11_4"
"Indexing chunks for book doc_11_4"
"Embedding chunks for book doc_11_5"
"Indexing chunks for book doc_11_4"
"Embedding chunks for book doc_11_5"
"Embedding chunks for book doc_11_5"
"Embedding chunks for book doc_11_5"
"Embedding chunks for book doc_11_5"
"Indexing chunks for book doc_11_5"
"Indexing chunks for book doc_11_5"
"Indexing chunks for book doc_11_5"
"Indexing chunks for book doc_11_5"
"Indexing chunks for book doc_11_5"
"Embedding chunks for book doc_11_3"
"Embedding chunks for book doc_11_3"
"Embedding chunks for book doc_11_3"
"Embedding chunks for book doc_11_3"
"Embedding chunks for book doc_11_3"
"Indexing chunks for book doc_11_3"
"Indexing chunks for book doc_11_3"
"Indexing chunks for book doc_11_3"
"Indexing chunks for book doc_11_3"
"Indexing chunks for book doc_11_3"
"Adding book doc_12_1"
"Adding book doc_12_2"
"Starting chunking for book doc_12_2"
"Adding book doc_12_3"
"Starting chunking for book doc_12_3"
"Adding book doc_12_4"
"Starting chunking for book doc_12_4"
"Adding book doc_12_5"
"Starting chunking for book doc_12_5"
"Starting chunking for book doc_12_1"
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_12_2"
"Embedding chunks for book doc_12_2"
"Embedding chunks for book doc_12_2"
"Embedding chunks for book doc_12_2"
"Embedding chunks for book doc_12_2"
"Indexing chunks for book doc_12_2"
"Indexing chunks for book doc_12_2"
"Indexing chunks for book doc_12_2"
"Indexing chunks for book doc_12_2"
"Indexing chunks for book doc_12_2"
"Embedding chunks for book doc_12_3"
"Embedding chunks for book doc_12_3"
"Embedding chunks for book doc_12_3"
"Embedding chunks for book doc_12_3"
"Embedding chunks for book doc_12_3"
"Indexing chunks for book doc_12_3"
"Indexing chunks for book doc_12_3"
"Indexing chunks for book doc_12_3"
"Indexing chunks for book doc_12_3"
"Indexing chunks for book doc_12_3"
"Embedding chunks for book doc_12_4"
"Embedding chunks for book doc_12_4"
"Embedding chunks for book doc_12_4"
"Embedding chunks for book doc_12_4"
"Embedding chunks for book doc_12_4"
"Indexing chunks for book doc_12_4"
"Indexing chunks for book doc_12_4"
"Indexing chunks for book doc_12_4"
"Indexing chunks for book doc_12_4"
"Indexing chunks for book doc_12_4"
"Embedding chunks for book doc_12_5"
"Embedding chunks for book doc_12_5"
"Embedding chunks for book doc_12_5"
"Embedding chunks for book doc_12_5"
"Embedding chunks for book doc_12_5"
"Indexing chunks for book doc_12_5"
"Indexing chunks for book doc_12_5"
"Embedding chunks for book doc_12_1"
"Indexing chunks for book doc_12_5"
"Indexing chunks for book doc_12_5"
"Indexing chunks for book doc_12_5"
"Embedding chunks for book doc_12_1"
"Embedding chunks for book doc_12_1"
"Embedding chunks for book doc_12_1"
"Embedding chunks for book doc_12_1"
"Indexing chunks for book doc_12_1"
"Indexing chunks for book doc_12_1"
"Indexing chunks for book doc_12_1"
"Indexing chunks for book doc_12_1"
"Indexing chunks for book doc_12_1"
"Adding book doc_13_1"
"Starting chunking for book doc_13_1"
"Adding book doc_13_2"
"Starting chunking for book doc_13_2"
"Adding book doc_13_3"
"Adding book doc_13_4"
"Starting chunking for book doc_13_3"
"Adding book doc_13_5"
"Starting chunking for book doc_13_4"
"Starting chunking for book doc_13_5"
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_13_3"
"Embedding chunks for book doc_13_3"
"Embedding chunks for book doc_13_3"
"Embedding chunks for book doc_13_4"
"Embedding chunks for book doc_13_3"
"Embedding chunks for book doc_13_4"
"Indexing chunks for book doc_13_3"
"Embedding chunks for book doc_13_4"
"Indexing chunks for book doc_13_3"
"Embedding chunks for book doc_13_4"
"Indexing chunks for book doc_13_3"
"Indexing chunks for book doc_13_4"
"Indexing chunks for book doc_13_3"
"Embedding chunks for book doc_13_3"
"Embedding chunks for book doc_13_4"
"Indexing chunks for book doc_13_4"
"Indexing chunks for book doc_13_4"
"Indexing chunks for book doc_13_3"
"Indexing chunks for book doc_13_4"
"Indexing chunks for book doc_13_4"
"Embedding chunks for book doc_13_1"
"Embedding chunks for book doc_13_1"
"Embedding chunks for book doc_13_1"
"Embedding chunks for book doc_13_1"
"Embedding chunks for book doc_13_1"
"Embedding chunks for book doc_13_5"
"Indexing chunks for book doc_13_1"
"Indexing chunks for book doc_13_1"
"Indexing chunks for book doc_13_1"
"Indexing chunks for book doc_13_1"
"Indexing chunks for book doc_13_1"
"Embedding chunks for book doc_13_2"
"Embedding chunks for book doc_13_2"
"Embedding chunks for book doc_13_5"
"Embedding chunks for book doc_13_5"
"Indexing chunks for book doc_13_5"
"Indexing chunks for book doc_13_2"
"Indexing chunks for book doc_13_5"
"Embedding chunks for book doc_13_2"
"Indexing chunks for book doc_13_2"
"Indexing chunks for book doc_13_5"
"Embedding chunks for book doc_13_5"
"Embedding chunks for book doc_13_2"
"Embedding chunks for book doc_13_5"
"Embedding chunks for book doc_13_2"
"Indexing chunks for book doc_13_2"
"Indexing chunks for book doc_13_5"
"Indexing chunks for book doc_13_5"
"Indexing chunks for book doc_13_2"
"Indexing chunks for book doc_13_2"
"Adding book doc_14_1"
"Starting chunking for book doc_14_1"
"Adding book doc_14_2"
"Adding book doc_14_3"
"Starting chunking for book doc_14_2"
"Starting chunking for book doc_14_3"
"Adding book doc_14_4"
"Starting chunking for book doc_14_4"
"Adding book doc_14_5"
"Starting chunking for book doc_14_5"
"Embedding chunks for book doc_14_3"
"Embedding chunks for book doc_14_3"
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_14_3"
"Embedding chunks for book doc_14_3"
"Embedding chunks for book doc_14_3"
"Indexing chunks for book doc_14_3"
"Indexing chunks for book doc_14_3"
"Indexing chunks for book doc_14_3"
"Indexing chunks for book doc_14_3"
"Indexing chunks for book doc_14_3"
"Embedding chunks for book doc_14_1"
"Embedding chunks for book doc_14_2"
"Embedding chunks for book doc_14_1"
"Embedding chunks for book doc_14_1"
"Embedding chunks for book doc_14_1"
"Indexing chunks for book doc_14_1"
"Indexing chunks for book doc_14_1"
"Indexing chunks for book doc_14_2"
"Indexing chunks for book doc_14_1"
"Embedding chunks for book doc_14_2"
"Embedding chunks for book doc_14_2"
"Embedding chunks for book doc_14_1"
"Embedding chunks for book doc_14_2"
"Embedding chunks for book doc_14_2"
"Indexing chunks for book doc_14_1"
"Indexing chunks for book doc_14_2"
"Embedding chunks for book doc_14_4"
"Indexing chunks for book doc_14_2"
"Embedding chunks for book doc_14_4"
"Indexing chunks for book doc_14_2"
"Indexing chunks for book doc_14_1"
"Indexing chunks for book doc_14_2"
"Embedding chunks for book doc_14_4"
"Embedding chunks for book doc_14_4"
"Embedding chunks for book doc_14_4"
"Embedding chunks for book doc_14_5"
"Indexing chunks for book doc_14_4"
"Indexing chunks for book doc_14_4"
"Indexing chunks for book doc_14_4"
"Indexing chunks for book doc_14_4"
"Indexing chunks for book doc_14_4"
"Embedding chunks for book doc_14_5"
"Embedding chunks for book doc_14_5"
"Embedding chunks for book doc_14_5"
"Embedding chunks for book doc_14_5"
"Indexing chunks for book doc_14_5"
"Indexing chunks for book doc_14_5"
"Indexing chunks for book doc_14_5"
"Indexing chunks for book doc_14_5"
"Indexing chunks for book doc_14_5"
"Adding book doc_15_1"
"Starting chunking for book doc_15_1"
"Adding book doc_15_2"
"Starting chunking for book doc_15_2"
"Adding book doc_15_3"
"Adding book doc_15_4"
"Starting chunking for book doc_15_3"
"Starting chunking for book doc_15_4"
"Adding book doc_15_5"
"Starting chunking for book doc_15_5"
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_15_3"
incoming demand : 1
"Embedding chunks for book doc_15_3"
"Embedding chunks for book doc_15_3"
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_15_3"
"Embedding chunks for book doc_15_1"
"Embedding chunks for book doc_15_1"
"Embedding chunks for book doc_15_1"
"Indexing chunks for book doc_15_3"
"Embedding chunks for book doc_15_1"
"Indexing chunks for book doc_15_3"
"Indexing chunks for book doc_15_3"
"Indexing chunks for book doc_15_3"
"Indexing chunks for book doc_15_1"
"Embedding chunks for book doc_15_1"
"Embedding chunks for book doc_15_3"
"Indexing chunks for book doc_15_3"
"Embedding chunks for book doc_15_5"
"Indexing chunks for book doc_15_1"
"Indexing chunks for book doc_15_1"
"Indexing chunks for book doc_15_1"
"Indexing chunks for book doc_15_1"
"Embedding chunks for book doc_15_5"
"Embedding chunks for book doc_15_5"
"Embedding chunks for book doc_15_5"
"Embedding chunks for book doc_15_5"
"Embedding chunks for book doc_15_2"
"Embedding chunks for book doc_15_2"
"Indexing chunks for book doc_15_5"
"Indexing chunks for book doc_15_5"
"Indexing chunks for book doc_15_5"
"Indexing chunks for book doc_15_5"
"Embedding chunks for book doc_15_2"
"Embedding chunks for book doc_15_2"
"Embedding chunks for book doc_15_2"
"Indexing chunks for book doc_15_5"
"Indexing chunks for book doc_15_2"
"Indexing chunks for book doc_15_2"
"Indexing chunks for book doc_15_2"
"Embedding chunks for book doc_15_4"
"Indexing chunks for book doc_15_2"
"Indexing chunks for book doc_15_2"
"Embedding chunks for book doc_15_4"
"Embedding chunks for book doc_15_4"
"Embedding chunks for book doc_15_4"
"Embedding chunks for book doc_15_4"
"Indexing chunks for book doc_15_4"
"Indexing chunks for book doc_15_4"
"Indexing chunks for book doc_15_4"
"Indexing chunks for book doc_15_4"
"Indexing chunks for book doc_15_4"
"Adding book doc_16_1"
"Starting chunking for book doc_16_1"
"Adding book doc_16_2"
"Starting chunking for book doc_16_2"
"Adding book doc_16_3"
"Starting chunking for book doc_16_3"
"Adding book doc_16_4"
"Starting chunking for book doc_16_4"
"Adding book doc_16_5"
"Starting chunking for book doc_16_5"
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_16_1"
"Embedding chunks for book doc_16_2"
"Embedding chunks for book doc_16_2"
"Embedding chunks for book doc_16_2"
"Embedding chunks for book doc_16_1"
"Indexing chunks for book doc_16_1"
"Indexing chunks for book doc_16_2"
"Embedding chunks for book doc_16_1"
"Indexing chunks for book doc_16_2"
"Indexing chunks for book doc_16_2"
"Embedding chunks for book doc_16_2"
"Embedding chunks for book doc_16_1"
"Indexing chunks for book doc_16_1"
"Embedding chunks for book doc_16_2"
"Embedding chunks for book doc_16_1"
"Embedding chunks for book doc_16_3"
"Embedding chunks for book doc_16_3"
"Embedding chunks for book doc_16_3"
"Embedding chunks for book doc_16_3"
"Embedding chunks for book doc_16_3"
"Indexing chunks for book doc_16_2"
"Indexing chunks for book doc_16_1"
"Indexing chunks for book doc_16_1"
"Indexing chunks for book doc_16_2"
"Indexing chunks for book doc_16_1"
"Embedding chunks for book doc_16_4"
"Indexing chunks for book doc_16_3"
"Embedding chunks for book doc_16_4"
"Indexing chunks for book doc_16_3"
"Indexing chunks for book doc_16_3"
"Indexing chunks for book doc_16_3"
"Indexing chunks for book doc_16_3"
"Embedding chunks for book doc_16_4"
"Embedding chunks for book doc_16_4"
"Embedding chunks for book doc_16_4"
"Indexing chunks for book doc_16_4"
"Indexing chunks for book doc_16_4"
"Indexing chunks for book doc_16_4"
"Indexing chunks for book doc_16_4"
"Embedding chunks for book doc_16_5"
"Embedding chunks for book doc_16_5"
"Embedding chunks for book doc_16_5"
"Embedding chunks for book doc_16_5"
"Indexing chunks for book doc_16_4"
"Embedding chunks for book doc_16_5"
"Indexing chunks for book doc_16_5"
"Indexing chunks for book doc_16_5"
"Indexing chunks for book doc_16_5"
"Indexing chunks for book doc_16_5"
"Indexing chunks for book doc_16_5"
"Adding book doc_17_1"
"Adding book doc_17_2"
"Adding book doc_17_3"
"Starting chunking for book doc_17_1"
"Starting chunking for book doc_17_2"
"Starting chunking for book doc_17_3"
"Adding book doc_17_4"
"Starting chunking for book doc_17_4"
"Adding book doc_17_5"
"Starting chunking for book doc_17_5"
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_17_1"
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_17_1"
incoming demand : 1
"Embedding chunks for book doc_17_1"
"Embedding chunks for book doc_17_1"
"Embedding chunks for book doc_17_1"
"Indexing chunks for book doc_17_1"
"Indexing chunks for book doc_17_1"
"Indexing chunks for book doc_17_1"
"Indexing chunks for book doc_17_1"
"Indexing chunks for book doc_17_1"
"Embedding chunks for book doc_17_5"
"Embedding chunks for book doc_17_2"
"Embedding chunks for book doc_17_5"
"Embedding chunks for book doc_17_5"
"Embedding chunks for book doc_17_2"
"Embedding chunks for book doc_17_2"
"Embedding chunks for book doc_17_2"
"Indexing chunks for book doc_17_5"
"Embedding chunks for book doc_17_5"
"Embedding chunks for book doc_17_5"
"Indexing chunks for book doc_17_5"
"Embedding chunks for book doc_17_2"
"Indexing chunks for book doc_17_2"
"Indexing chunks for book doc_17_2"
"Indexing chunks for book doc_17_5"
"Embedding chunks for book doc_17_3"
"Indexing chunks for book doc_17_2"
"Embedding chunks for book doc_17_3"
"Indexing chunks for book doc_17_2"
"Indexing chunks for book doc_17_2"
"Indexing chunks for book doc_17_5"
"Embedding chunks for book doc_17_3"
"Indexing chunks for book doc_17_5"
"Embedding chunks for book doc_17_3"
"Embedding chunks for book doc_17_3"
"Indexing chunks for book doc_17_3"
"Indexing chunks for book doc_17_3"
"Indexing chunks for book doc_17_3"
"Indexing chunks for book doc_17_3"
"Indexing chunks for book doc_17_3"
"Embedding chunks for book doc_17_4"
"Embedding chunks for book doc_17_4"
"Embedding chunks for book doc_17_4"
"Embedding chunks for book doc_17_4"
"Embedding chunks for book doc_17_4"
"Indexing chunks for book doc_17_4"
"Indexing chunks for book doc_17_4"
"Indexing chunks for book doc_17_4"
"Indexing chunks for book doc_17_4"
"Indexing chunks for book doc_17_4"
"Adding book doc_18_1"
"Adding book doc_18_2"
"Starting chunking for book doc_18_1"
"Starting chunking for book doc_18_2"
"Adding book doc_18_3"
"Starting chunking for book doc_18_3"
"Adding book doc_18_4"
"Adding book doc_18_5"
"Starting chunking for book doc_18_4"
"Starting chunking for book doc_18_5"
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_18_2"
"Embedding chunks for book doc_18_2"
"Embedding chunks for book doc_18_2"
"Embedding chunks for book doc_18_2"
"Embedding chunks for book doc_18_2"
"Indexing chunks for book doc_18_2"
"Indexing chunks for book doc_18_2"
"Embedding chunks for book doc_18_1"
"Indexing chunks for book doc_18_2"
"Indexing chunks for book doc_18_2"
"Indexing chunks for book doc_18_2"
"Embedding chunks for book doc_18_3"
"Embedding chunks for book doc_18_3"
"Embedding chunks for book doc_18_1"
"Embedding chunks for book doc_18_3"
"Indexing chunks for book doc_18_3"
"Indexing chunks for book doc_18_1"
"Indexing chunks for book doc_18_3"
"Indexing chunks for book doc_18_1"
"Indexing chunks for book doc_18_3"
"Embedding chunks for book doc_18_1"
"Embedding chunks for book doc_18_1"
"Embedding chunks for book doc_18_3"
"Embedding chunks for book doc_18_3"
"Embedding chunks for book doc_18_1"
"Indexing chunks for book doc_18_3"
"Indexing chunks for book doc_18_1"
"Indexing chunks for book doc_18_1"
"Embedding chunks for book doc_18_5"
"Embedding chunks for book doc_18_5"
"Embedding chunks for book doc_18_5"
"Indexing chunks for book doc_18_3"
"Embedding chunks for book doc_18_5"
"Indexing chunks for book doc_18_1"
"Embedding chunks for book doc_18_5"
"Indexing chunks for book doc_18_5"
"Indexing chunks for book doc_18_5"
"Indexing chunks for book doc_18_5"
"Indexing chunks for book doc_18_5"
"Indexing chunks for book doc_18_5"
"Embedding chunks for book doc_18_4"
"Embedding chunks for book doc_18_4"
"Embedding chunks for book doc_18_4"
"Embedding chunks for book doc_18_4"
"Embedding chunks for book doc_18_4"
"Indexing chunks for book doc_18_4"
"Indexing chunks for book doc_18_4"
"Indexing chunks for book doc_18_4"
"Indexing chunks for book doc_18_4"
"Indexing chunks for book doc_18_4"
"Adding book doc_19_1"
"Adding book doc_19_2"
"Adding book doc_19_3"
"Starting chunking for book doc_19_1"
"Starting chunking for book doc_19_3"
"Adding book doc_19_4"
"Starting chunking for book doc_19_4"
"Adding book doc_19_5"
"Starting chunking for book doc_19_2"
"Starting chunking for book doc_19_5"
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_19_1"
"Embedding chunks for book doc_19_3"
"Embedding chunks for book doc_19_1"
"Embedding chunks for book doc_19_3"
"Embedding chunks for book doc_19_1"
"Indexing chunks for book doc_19_1"
"Indexing chunks for book doc_19_3"
"Indexing chunks for book doc_19_1"
"Indexing chunks for book doc_19_3"
"Indexing chunks for book doc_19_1"
"Embedding chunks for book doc_19_3"
"Embedding chunks for book doc_19_1"
"Embedding chunks for book doc_19_1"
"Embedding chunks for book doc_19_3"
"Embedding chunks for book doc_19_3"
"Embedding chunks for book doc_19_5"
"Indexing chunks for book doc_19_3"
"Embedding chunks for book doc_19_4"
"Indexing chunks for book doc_19_1"
"Embedding chunks for book doc_19_5"
"Indexing chunks for book doc_19_3"
"Embedding chunks for book doc_19_5"
"Indexing chunks for book doc_19_3"
"Indexing chunks for book doc_19_1"
"Embedding chunks for book doc_19_5"
"Indexing chunks for book doc_19_5"
"Indexing chunks for book doc_19_4"
"Indexing chunks for book doc_19_5"
"Indexing chunks for book doc_19_5"
"Indexing chunks for book doc_19_5"
"Embedding chunks for book doc_19_5"
"Embedding chunks for book doc_19_4"
"Embedding chunks for book doc_19_4"
"Embedding chunks for book doc_19_4"
"Embedding chunks for book doc_19_4"
"Embedding chunks for book doc_19_2"
"Indexing chunks for book doc_19_5"
"Embedding chunks for book doc_19_2"
"Embedding chunks for book doc_19_2"
"Indexing chunks for book doc_19_4"
"Embedding chunks for book doc_19_2"
"Indexing chunks for book doc_19_4"
"Embedding chunks for book doc_19_2"
"Indexing chunks for book doc_19_4"
"Indexing chunks for book doc_19_4"
"Indexing chunks for book doc_19_2"
"Indexing chunks for book doc_19_2"
"Indexing chunks for book doc_19_2"
"Indexing chunks for book doc_19_2"
"Indexing chunks for book doc_19_2"
"Adding book doc_20_1"
"Adding book doc_20_2"
"Adding book doc_20_3"
"Adding book doc_20_4"
"Adding book doc_20_5"
"Starting chunking for book doc_20_1"
"Starting chunking for book doc_20_3"
"Starting chunking for book doc_20_2"
"Starting chunking for book doc_20_5"
"Starting chunking for book doc_20_4"
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
incoming demand : 1
"Embedding chunks for book doc_20_5"
"Embedding chunks for book doc_20_5"
"Embedding chunks for book doc_20_5"
"Embedding chunks for book doc_20_5"
"Embedding chunks for book doc_20_5"
"Indexing chunks for book doc_20_5"
"Indexing chunks for book doc_20_5"
"Indexing chunks for book doc_20_5"
"Indexing chunks for book doc_20_5"
"Embedding chunks for book doc_20_3"
"Embedding chunks for book doc_20_3"
"Embedding chunks for book doc_20_3"
"Embedding chunks for book doc_20_3"
"Indexing chunks for book doc_20_5"
"Embedding chunks for book doc_20_3"
"Embedding chunks for book doc_20_1"
"Indexing chunks for book doc_20_3"
"Embedding chunks for book doc_20_1"
"Embedding chunks for book doc_20_1"
"Embedding chunks for book doc_20_1"
"Embedding chunks for book doc_20_1"
"Indexing chunks for book doc_20_3"
"Indexing chunks for book doc_20_3"
"Indexing chunks for book doc_20_3"
"Indexing chunks for book doc_20_3"
"Indexing chunks for book doc_20_1"
"Indexing chunks for book doc_20_1"
"Indexing chunks for book doc_20_1"
"Indexing chunks for book doc_20_1"
"Indexing chunks for book doc_20_1"
"Embedding chunks for book doc_20_2"
"Embedding chunks for book doc_20_2"
"Embedding chunks for book doc_20_2"
"Embedding chunks for book doc_20_2"
"Embedding chunks for book doc_20_2"
"Indexing chunks for book doc_20_2"
"Indexing chunks for book doc_20_2"
"Indexing chunks for book doc_20_2"
"Indexing chunks for book doc_20_2"
"Indexing chunks for book doc_20_2"
"Embedding chunks for book doc_20_4"
"Embedding chunks for book doc_20_4"
"Embedding chunks for book doc_20_4"
"Embedding chunks for book doc_20_4"
"Embedding chunks for book doc_20_4"
"Indexing chunks for book doc_20_4"
"Indexing chunks for book doc_20_4"
"Indexing chunks for book doc_20_4"
"Indexing chunks for book doc_20_4"
"Indexing chunks for book doc_20_4"
[:ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok]
state = SimulIngestion.Dashboard.get_state()
  
data =
  state  
  |> Map.get(:freq)
  |> Enum.map(fn {time, nb} -> %{"time" => time / 1000, "nb" => nb} end)

speed =
  state
  |> Map.get(:embedding_speed)
  |> Enum.map(fn {time, embed_sec} -> %{"time" => time, "embed_sec" => embed_sec} end)

state.docs
{100, 100, 100}
VegaLite.new(title: "#docs / #sec")
|> VegaLite.data_from_values(data, only: ["time", "nb"])
|> VegaLite.mark(:bar)
|> VegaLite.encode_field(:x, "time", type: :quantitative, bin: [maxbins: 40])
|> VegaLite.encode_field(:y, "nb", type: :quantitative)
{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"nb":3,"time":0.312},{"nb":4,"time":0.313},{"nb":5,"time":0.314},{"nb":6,"time":0.613},{"nb":7,"time":0.614},{"nb":7,"time":0.615},{"nb":2,"time":0.616},{"nb":6,"time":0.914},{"nb":14,"time":0.915},{"nb":4,"time":0.916},{"nb":5,"time":1.215},{"nb":8,"time":1.216},{"nb":6,"time":1.217},{"nb":2,"time":1.218},{"nb":9,"time":1.516},{"nb":8,"time":1.517},{"nb":3,"time":1.518},{"nb":1,"time":1.519}]},"encoding":{"x":{"bin":{"maxbins":40},"field":"time","type":"quantitative"},"y":{"field":"nb","type":"quantitative"}},"mark":"bar","title":"#docs / #sec"}
VegaLite.new()
|> VegaLite.data_from_values(speed, only: ["time", "embed_sec"])
|> VegaLite.mark(:line)
|> VegaLite.encode_field(:x, "time", type: :temporal)
|> VegaLite.encode_field(:y, "embed_sec", type: :quantitative)
{"$schema":"https://vega.github.io/schema/vega-lite/v5.json","data":{"values":[{"embed_sec":10,"time":"2024-10-14T16:06:26.026793Z"},{"embed_sec":20,"time":"2024-10-14T16:06:24.930785Z"},{"embed_sec":20,"time":"2024-10-14T16:06:23.724788Z"},{"embed_sec":10,"time":"2024-10-14T16:06:22.025557Z"},{"embed_sec":19,"time":"2024-10-14T16:06:20.927929Z"},{"embed_sec":20,"time":"2024-10-14T16:06:19.722829Z"},{"embed_sec":10,"time":"2024-10-14T16:06:18.023130Z"},{"embed_sec":19,"time":"2024-10-14T16:06:16.926149Z"},{"embed_sec":20,"time":"2024-10-14T16:06:15.720832Z"},{"embed_sec":10,"time":"2024-10-14T16:06:14.019717Z"},{"embed_sec":19,"time":"2024-10-14T16:06:12.922077Z"},{"embed_sec":20,"time":"2024-10-14T16:06:11.717717Z"},{"embed_sec":10,"time":"2024-10-14T16:06:10.018052Z"},{"embed_sec":20,"time":"2024-10-14T16:06:08.919887Z"},{"embed_sec":20,"time":"2024-10-14T16:06:07.716044Z"},{"embed_sec":10,"time":"2024-10-14T16:06:06.015867Z"},{"embed_sec":20,"time":"2024-10-14T16:06:04.918033Z"},{"embed_sec":20,"time":"2024-10-14T16:06:03.713800Z"},{"embed_sec":10,"time":"2024-10-14T16:06:02.013994Z"},{"embed_sec":20,"time":"2024-10-14T16:06:00.915836Z"},{"embed_sec":20,"time":"2024-10-14T16:05:59.711758Z"},{"embed_sec":10,"time":"2024-10-14T16:05:58.011741Z"},{"embed_sec":20,"time":"2024-10-14T16:05:56.913698Z"},{"embed_sec":20,"time":"2024-10-14T16:05:55.709837Z"},{"embed_sec":10,"time":"2024-10-14T16:05:54.010036Z"},{"embed_sec":20,"time":"2024-10-14T16:05:52.912722Z"},{"embed_sec":20,"time":"2024-10-14T16:05:51.708858Z"},{"embed_sec":10,"time":"2024-10-14T16:05:50.008848Z"},{"embed_sec":20,"time":"2024-10-14T16:05:48.909676Z"},{"embed_sec":0.0,"time":"2024-10-14T16:05:47.704798Z"}]},"encoding":{"x":{"field":"time","type":"temporal"},"y":{"field":"embed_sec","type":"quantitative"}},"mark":"line"}
Embedding service at max capacity