Chapter 18: Beyond the Sigmoid

Mix.install(
  [
    {:exla, "~> 0.5"},
    {:nx, "~> 0.5"},
    {:axon, "~> 0.5"},
    {:kino, "~> 0.8.1"},
    {:kino_vega_lite, "~> 0.1.7"},
    {:vega_lite, "~> 0.1.6"}
  ],
  config: [nx: [default_backend: EXLA.Backend]]
)

ReLU

relu_fn = fn z ->
  if z <= 0 do
    0
  else
    z
  end
end

dataset = Enum.map(-6..6, fn x -> %{x: x, y: relu_fn.(x)} end)

VegaLite.new(width: 600, height: 400, title: "ReLU(z)")
|> VegaLite.data_from_values(dataset, only: ["x", "y"])
|> VegaLite.mark(:line)
|> VegaLite.encode_field(:x, "x", type: :quantitative)
|> VegaLite.encode_field(:y, "y", type: :quantitative)

Leaky ReLU

leaky_relu_fn = fn z, alpha ->
  if z <= 0 do
    z * alpha
  else
    z
  end
end

alpha = 0.02
dataset = Enum.map(-6..6, fn x -> %{x: x, y: leaky_relu_fn.(x, alpha)} end)

VegaLite.new(width: 600, height: 400, title: "Leaky ReLU(z, alpha)")
|> VegaLite.data_from_values(dataset, only: ["x", "y"])
|> VegaLite.mark(:line)
|> VegaLite.encode_field(:x, "x", type: :quantitative)
|> VegaLite.encode_field(:y, "y", type: :quantitative)