High Performance Polling

Scale SNMP polling to hundreds or thousands of devices with SnmpKit's shared-socket engine.

Setup

Mix.install([
  {:snmpkit, "~> 2.0"}
])

alias SnmpKit.{SNMP, Sim}
alias SnmpKit.SnmpSim.ProfileLoader

# A free UDP port on the loopback interface, so the notebook runs anywhere
free_port = fn ->
  {:ok, socket} = :gen_udp.open(0, ip: {127, 0, 0, 1})
  {:ok, port} = :inet.port(socket)
  :gen_udp.close(socket)
  port
end

IO.puts("High performance modules ready!")

The shared-socket engine

Multi-target calls (SNMP.get_multi/2, SNMP.walk_multi/2, SNMP.get_bulk_multi/2, SNMP.walk_table_multi/2) are run by SnmpKit.SnmpMgr.Multi on top of SnmpKit.SnmpMgr.Engine:

• One UDP socket with a large receive buffer, shared by every request
• Direct sending - no GenServer in the send path
• Centralized response correlation by request id
• Configurable concurrency (max_concurrent:) and per-request options

Nothing has to be started by hand; the engine comes up on the first call. Results are a plain list in request order, one {:ok, varbinds} or {:error, reason} per request, and the varbinds are the same enriched maps (oid, oid_list, name, type, value, formatted) that every single-target call returns.

Create a Device Fleet

Let's simulate a realistic network with 20 devices, each on its own free port of the loopback interface:

# Create 20 diverse devices
device_count = 20

devices = for i <- 1..device_count do
  # Vary device types
  {device_type, description, if_count} = case rem(i, 4) do
    0 -> {:switch, "Cisco Catalyst 2960-#{24 + rem(i, 3) * 24}TT", 24 + rem(i, 3) * 24}
    1 -> {:router, "Cisco ISR 4331", 4}
    2 -> {:cable_modem, "ARRIS SB8200 DOCSIS 3.1", 2}
    3 -> {:access_point, "Ubiquiti UniFi AP-AC-Pro", 3}
  end

  # Build OID map
  oids = %{
    "1.3.6.1.2.1.1.1.0" => description,
    "1.3.6.1.2.1.1.2.0" => "1.3.6.1.4.1.9.1.#{100 + i}",
    "1.3.6.1.2.1.1.3.0" => %{type: "TimeTicks", value: :rand.uniform(864_000_00)},
    "1.3.6.1.2.1.1.4.0" => "netops@company.com",
    "1.3.6.1.2.1.1.5.0" => "#{device_type}-#{String.pad_leading("#{i}", 3, "0")}",
    "1.3.6.1.2.1.1.6.0" => "Rack #{div(i - 1, 5) + 1}, Unit #{rem(i - 1, 5) + 1}",
    "1.3.6.1.2.1.2.1.0" => if_count
  }

  # Add interfaces
  oids = Enum.reduce(1..if_count, oids, fn port, acc ->
    Map.merge(acc, %{
      "1.3.6.1.2.1.2.2.1.1.#{port}" => port,
      "1.3.6.1.2.1.2.2.1.2.#{port}" => "eth#{port - 1}",
      "1.3.6.1.2.1.2.2.1.3.#{port}" => 6,
      "1.3.6.1.2.1.2.2.1.5.#{port}" => %{type: "Gauge32", value: 1_000_000_000},
      "1.3.6.1.2.1.2.2.1.8.#{port}" => if(:rand.uniform() > 0.1, do: 1, else: 2),
      "1.3.6.1.2.1.2.2.1.10.#{port}" => %{type: "Counter32", value: :rand.uniform(4_000_000_000)},
      "1.3.6.1.2.1.2.2.1.16.#{port}" => %{type: "Counter32", value: :rand.uniform(2_000_000_000)}
    })
  end)

  port = free_port.()
  {:ok, profile} = ProfileLoader.load_profile(device_type, {:manual, oids})
  {:ok, device} = Sim.start_device(profile, port: port, bind_address: "127.0.0.1")

  %{
    port: port,
    target: "127.0.0.1:#{port}",
    type: device_type,
    device: device
  }
end

IO.puts("Created #{length(devices)} devices on 127.0.0.1")
IO.puts("")

# Show device distribution
devices
|> Enum.group_by(& &1.type)
|> Enum.each(fn {type, devs} ->
  IO.puts("  #{type}: #{length(devs)} devices")
end)

Multi-Target GET

Poll all devices simultaneously:

targets = Enum.map(devices, & &1.target)

# Build requests for sysDescr from all devices
requests = Enum.map(targets, &{&1, "sysDescr.0"})

# Time the operation
{time_us, results} = :timer.tc(fn ->
  SNMP.get_multi(requests)
end)

success_count = Enum.count(results, &match?({:ok, _}, &1))

IO.puts("GET #{length(requests)} devices:")
IO.puts("  Time: #{div(time_us, 1000)} ms")
IO.puts("  Success: #{success_count}/#{length(results)}")
IO.puts("  Rate: #{Float.round(length(requests) / (time_us / 1_000_000), 1)} req/sec")

Each entry is the result for the request at the same position. A GET answers with one enriched varbind:

[{:ok, [first_varbind]} | _] = results
first_varbind

Multi-Target GET with Multiple OIDs

Get several values from each device:

targets = Enum.map(devices, & &1.target)

# Request multiple OIDs per device
oids = ["sysDescr.0", "sysName.0", "sysUpTime.0", "ifNumber.0"]

requests = for target <- targets, oid <- oids do
  {target, oid}
end

{time_us, results} = :timer.tc(fn ->
  SNMP.get_multi(requests)
end)

success_count = Enum.count(results, &match?({:ok, _}, &1))

IO.puts("GET #{length(requests)} values (#{length(targets)} devices x #{length(oids)} OIDs):")
IO.puts("  Time: #{div(time_us, 1000)} ms")
IO.puts("  Success: #{success_count}/#{length(results)}")
IO.puts("  Rate: #{Float.round(length(requests) / (time_us / 1_000_000), 1)} req/sec")

Concurrency Control

Adjust max_concurrent to control parallel requests. On the loopback interface every level finishes in a few milliseconds; the difference shows on a real network, where each in-flight request is waiting on a round trip.

targets = Enum.map(devices, & &1.target)
requests = Enum.map(targets, &{&1, "sysDescr.0"})

IO.puts("Comparing concurrency levels:\n")

for max_concurrent <- [1, 5, 10, 20] do
  {time_us, results} = :timer.tc(fn ->
    SNMP.get_multi(requests, max_concurrent: max_concurrent)
  end)

  success = Enum.count(results, &match?({:ok, _}, &1))
  rate = Float.round(length(requests) / (time_us / 1_000_000), 1)

  IO.puts("  max_concurrent: #{max_concurrent}")
  IO.puts("    Time: #{div(time_us, 1000)} ms, Rate: #{rate} req/sec, Success: #{success}/#{length(results)}")
  IO.puts("")
end

Multi-Target WALK

Walk entire subtrees from multiple devices:

targets = Enum.map(devices, & &1.target)

# Walk system group from all devices
walk_requests = Enum.map(targets, &{&1, "system"})

{time_us, results} = :timer.tc(fn ->
  SNMP.walk_multi(walk_requests)
end)

success_results = Enum.filter(results, &match?({:ok, _}, &1))
total_oids = Enum.reduce(success_results, 0, fn {:ok, data}, acc -> acc + length(data) end)

IO.puts("WALK system from #{length(targets)} devices:")
IO.puts("  Time: #{div(time_us, 1000)} ms")
IO.puts("  Success: #{length(success_results)}/#{length(results)} devices")
IO.puts("  Total OIDs: #{total_oids}")

Walk Interface Tables

Walk larger tables from all devices. Three options bound the time a request may take, on every path (single-target, multi-target, streams):

• timeout: - how long to wait for one PDU (default 5000 ms)
• retries: - how many times a timed-out PDU is re-sent (default 1)
• walk_timeout: - a cap on a whole walk, which sends many PDUs (default max(timeout * 10, 1_200_000) ms)

A walk of a big table needs walk_timeout:, not a larger timeout:.

targets = Enum.map(devices, & &1.target)

walk_requests = Enum.map(targets, &{&1, "interfaces"})

{time_us, results} = :timer.tc(fn ->
  SNMP.walk_multi(walk_requests, walk_timeout: 30_000)
end)

success_results = Enum.filter(results, &match?({:ok, _}, &1))
total_oids = Enum.reduce(success_results, 0, fn {:ok, data}, acc -> acc + length(data) end)

IO.puts("WALK interfaces from #{length(targets)} devices:")
IO.puts("  Time: #{div(time_us, 1000)} ms")
IO.puts("  Success: #{length(success_results)}/#{length(results)} devices")
IO.puts("  Total OIDs: #{total_oids}")
IO.puts("  Avg OIDs/device: #{if length(success_results) > 0, do: div(total_oids, length(success_results)), else: 0}")

Per-Request Options

A request may carry its own options as a third tuple element. They win over the options of the call, so a slow WAN device can get a longer timeout: and more retries: without slowing the rest of the fleet:

targets = Enum.map(devices, & &1.target)

# Mix of normal and custom-timeout requests
requests = targets
|> Enum.with_index()
|> Enum.map(fn {target, i} ->
  if rem(i, 5) == 0 do
    # Every 5th request may take up to 15 s per PDU and is re-sent twice
    {target, "sysDescr.0", timeout: 15_000, retries: 2}
  else
    {target, "sysDescr.0"}
  end
end)

{time_us, results} = :timer.tc(fn ->
  SNMP.get_multi(requests, timeout: 2_000, retries: 0)
end)

IO.puts("GET with mixed timeouts:")
IO.puts("  Time: #{div(time_us, 1000)} ms")
IO.puts("  Success: #{Enum.count(results, &match?({:ok, _}, &1))}/#{length(results)}")

An unreachable device costs timeout * (retries + 1) and comes back as {:error, :timeout} in its slot; the other results are unaffected:

nobody = "127.0.0.1:#{free_port.()}"

SNMP.get_multi([{nobody, "sysDescr.0"}, {hd(targets), "sysDescr.0"}], timeout: 300, retries: 1)
|> Enum.map(fn
  {:ok, [%{formatted: v}]} -> {:ok, v}
  error -> error
end)

Return Formats

Choose how results are returned:

targets = Enum.take(Enum.map(devices, & &1.target), 5)
requests = Enum.map(targets, &{&1, "sysName.0"})

IO.puts("Return format comparison:\n")

# Default: list
results_list = SNMP.get_multi(requests, return_format: :list)
IO.puts("  :list format:")
Enum.each(results_list, fn
  {:ok, [%{formatted: v} | _]} -> IO.puts("    #{v}")
  {:error, e} -> IO.puts("    Error: #{inspect(e)}")
end)

IO.puts("")

# With targets
results_with = SNMP.get_multi(requests, return_format: :with_targets)
IO.puts("  :with_targets format:")
Enum.each(results_with, fn
  {target, oid, {:ok, [%{formatted: v} | _]}} -> IO.puts("    #{target} #{oid}: #{v}")
  {target, oid, {:error, _e}} -> IO.puts("    #{target} #{oid}: Error")
end)

IO.puts("")

# As map
results_map = SNMP.get_multi(requests, return_format: :map)
IO.puts("  :map format:")
Enum.each(results_map, fn
  {{target, oid}, {:ok, [%{formatted: v} | _]}} -> IO.puts("    {#{target}, #{oid}} => #{v}")
  {{target, oid}, {:error, _}} -> IO.puts("    {#{target}, #{oid}} => Error")
end)

Bulk Operations at Scale

Use GETBULK for efficient table retrieval. GETBULK exists since SNMPv2c, so get_bulk_multi runs over v2c (the default) or v3 and refuses version: :v1:

targets = Enum.map(devices, & &1.target)

bulk_requests = Enum.map(targets, &{&1, "ifDescr"})

{time_us, results} = :timer.tc(fn ->
  SNMP.get_bulk_multi(bulk_requests, max_repetitions: 20)
end)

success_results = Enum.filter(results, &match?({:ok, _}, &1))
total_oids = Enum.reduce(success_results, 0, fn {:ok, data}, acc -> acc + length(data) end)

IO.puts("GETBULK ifDescr from #{length(targets)} devices:")
IO.puts("  Time: #{div(time_us, 1000)} ms")
IO.puts("  Success: #{length(success_results)}/#{length(results)} devices")
IO.puts("  Total OIDs: #{total_oids}")

Polling Loop Pattern

A typical monitoring pattern:

targets = Enum.map(devices, & &1.target)

# Define what to poll
poll_oids = ["sysUpTime.0", "ifInOctets.1", "ifOutOctets.1"]

# Build all requests
requests = for target <- targets, oid <- poll_oids, do: {target, oid}

# Simulate a few poll cycles
IO.puts("Simulating 3 poll cycles:\n")

for cycle <- 1..3 do
  {time_us, results} = :timer.tc(fn ->
    SNMP.get_multi(requests, max_concurrent: 20)
  end)

  success = Enum.count(results, &match?({:ok, _}, &1))
  rate = Float.round(length(requests) / (time_us / 1_000_000), 1)

  IO.puts("Cycle #{cycle}: #{success}/#{length(requests)} success, #{div(time_us, 1000)}ms, #{rate} req/sec")

  # In real code, you'd process results here
  Process.sleep(500)
end

Turning the octet counters from two cycles into bits per second is the subject of the telemetry and rates notebook.

Engine Health

The engine reports on itself. health_check/0 grades the depth of its mailbox (:healthy, :warning, :critical, or :error when the socket is gone) and get_stats/0 has the correlation counters and average response time. Both are maps; every engine call returns {:error, :services_not_started} instead of crashing if the engine is not running.

SnmpKit.SnmpMgr.Engine.health_check()

%{metrics: metrics, pending_requests: pending} = SnmpKit.SnmpMgr.Engine.get_stats()

IO.puts("pending: #{pending}")
IO.puts("completed: #{metrics.requests_completed}, timed out: #{metrics.requests_timeout}")
IO.puts("avg response: #{Float.round(metrics.avg_response_time * 1.0, 2)} ms")

Every multi-target call also emits [:snmpkit, :multi, :stop] with its ok_count and error_count, and every request the engine gives up on emits [:snmpkit, :engine, :timeout].

Performance Tips

1. Tune max_concurrent - Start with 10-20, increase based on network capacity
2. Use bulk operations - GETBULK is faster than multiple GETs for tables
3. Batch requests - Combine multiple OIDs in single get_multi calls
4. Set appropriate timeouts - timeout: and retries: per PDU, walk_timeout: per walk
5. Handle errors gracefully - Some devices may be unreachable

IO.puts("Performance guidelines:")
IO.puts("")
IO.puts("  Devices    max_concurrent    Expected")
IO.puts("  -------    --------------    --------")
IO.puts("  10-50      5-10              Fast")
IO.puts("  50-200     10-20             Good")
IO.puts("  200-1000   20-50             Monitor carefully")
IO.puts("  1000+      50-100            Test thoroughly")

Cleanup

Devices started with Sim.start_device/2 are linked to the process that started them; stopping them explicitly frees the ports right away.

Enum.each(devices, &SnmpKit.SnmpSim.stop_device(&1.device))
IO.puts("Stopped #{length(devices)} devices")

Next Steps

• Telemetry and Rates - Watch the engine's events and turn counters into rates
• SNMPv3 - The same multi-target calls with USM users
• Device Simulation - More ways to build the fleet you poll