Packages
Multi-surface application runtime for Elixir. One TEA module renders to terminal, browser (LiveView), SSH, and MCP (agents). 30+ widgets, flexbox + CSS grid, AI agent runtime, distributed swarm with CRDTs, time-travel debugging, session recording, sandboxed REPL, and agentic commerce.
Current section
Files
Jump to
Current section
Files
lib/raxol/plugins/visualization/image_renderer.ex
defmodule Raxol.Plugins.Visualization.ImageRenderer do
@moduledoc """
Handles rendering logic for image visualization within the VisualizationPlugin.
Supports both sixel and kitty protocols for terminal image rendering.
"""
require Raxol.Core.Runtime.Log
alias Raxol.Terminal.Cell
alias Raxol.Plugins.Visualization.DrawingUtils
@doc """
Public entry point for rendering image content.
Handles bounds checking and calls the internal drawing logic.
Expects bounds map: %{width: w, height: h}.
"""
def render_image_content(
data,
opts,
%{width: width, height: height} = bounds,
state
) do
title = Map.get(opts, :title, "Image")
protocol = Map.get(opts, :protocol, detect_protocol(state))
case width < 1 or height < 1 do
true ->
Raxol.Core.Runtime.Log.warning_with_context(
"[ImageRenderer] Bounds too small for image rendering: #{inspect(bounds)}",
%{}
)
[]
false ->
case Raxol.Core.ErrorHandling.safe_call(fn ->
case protocol do
:sixel -> render_sixel(data, bounds, opts)
:kitty -> render_kitty(data, bounds, opts)
_ -> draw_placeholder(data, title, bounds)
end
end) do
{:ok, result} ->
result
{:error, e} ->
Raxol.Core.Runtime.Log.error(
"[ImageRenderer] Error rendering image: #{inspect(e)}"
)
DrawingUtils.draw_box_with_text("[Render Error]", bounds)
end
end
end
defp detect_protocol(state) do
case {supports_kitty?(state), supports_sixel?(state)} do
{true, _} -> :kitty
{_, true} -> :sixel
{false, false} -> :placeholder
end
end
defp supports_kitty?(state) do
# Check for kitty protocol support
term_program = get_in(state, [:terminal, :program])
term_program == "kitty" or String.contains?(term_program || "", "kitty")
end
defp supports_sixel?(state) do
# Check for sixel support
_term_program = get_in(state, [:terminal, :program])
term_features = get_in(state, [:terminal, :features]) || []
"sixel" in term_features
end
defp render_sixel(data, bounds, opts) do
# Check if data is already a Sixel sequence (starts with DCS)
case is_sixel_sequence?(data) do
true ->
# Data is already Sixel - use it directly
create_sixel_cells(data, bounds)
false ->
# Try to load and convert image data to Sixel
case load_image_data(data) do
{:ok, image_data} ->
# Convert image to sixel format
sixel_data = convert_to_sixel(image_data, bounds)
# Create cells with sixel escape sequence
create_sixel_cells(sixel_data, bounds)
{:error, reason} ->
Raxol.Core.Runtime.Log.error(
"[ImageRenderer] Failed to load image: #{inspect(reason)}"
)
draw_placeholder(data, Map.get(opts, :title, "Image"), bounds)
end
end
end
@doc false
defp is_sixel_sequence?(data) when is_binary(data) do
# Check for DCS Sixel start sequence
String.starts_with?(data, "\e[") or String.starts_with?(data, "\eP")
end
defp is_sixel_sequence?(_), do: false
defp render_kitty(data, bounds, opts) do
case load_image_data(data) do
{:ok, image_data} ->
# Convert image to kitty format
kitty_data = convert_to_kitty(image_data, bounds)
# Create cells with kitty escape sequence
create_kitty_cells(kitty_data, bounds)
{:error, reason} ->
Raxol.Core.Runtime.Log.error(
"[ImageRenderer] Failed to load image: #{inspect(reason)}"
)
draw_placeholder(data, Map.get(opts, :title, "Image"), bounds)
end
end
defp load_image_data(data) when is_binary(data) do
# First try as file path, then as raw data
case File.read(data) do
{:ok, content} -> {:ok, content}
# Assume it's raw image data
{:error, _reason} -> {:ok, data}
end
end
defp load_image_data(_), do: {:error, :invalid_data}
defp convert_to_sixel(image_data, bounds) do
# Decode and resize image to fit bounds
with {:ok, image} <- decode_image(image_data),
resized_image <- resize_image(image, bounds),
sixel_data <- encode_sixel(resized_image) do
sixel_data
else
_ -> "Failed to convert image to sixel format"
end
end
defp decode_image(data) do
# Use Mogrify to decode image data
case Raxol.Core.ErrorHandling.safe_call(fn ->
Mogrify.open(data)
end) do
{:ok, image} -> {:ok, image}
{:error, e} -> {:error, Exception.message(e)}
end
end
defp resize_image(image, %{width: width, height: height}) do
# Resize image to fit terminal bounds
image
|> Mogrify.resize("#{width}x#{height}")
|> Mogrify.format("png")
|> Mogrify.save()
end
defp encode_sixel(_image) do
# For now, we rely on external Sixel encoders or pre-generated Sixel data
# A full implementation would convert Mogrify image to pixel data
# and build a SixelGraphics state with pixel_buffer, then encode it
# This is a simple test pattern for demonstration
# In practice, users should provide Sixel-encoded data or use external tools
state = Raxol.Terminal.ANSI.SixelGraphics.new(10, 10)
# Create a simple pattern in pixel buffer
pixel_buffer =
for x <- 0..9, y <- 0..9, into: %{} do
{{x, y}, rem(x + y, 4)}
end
state_with_pixels = %{state | pixel_buffer: pixel_buffer}
# Encode to Sixel format
Raxol.Terminal.ANSI.SixelGraphics.encode(state_with_pixels)
end
defp convert_to_kitty(image_data, bounds) do
with {:ok, image} <- decode_image(image_data),
resized_image <- resize_image(image, bounds) do
# Convert to base64 and create kitty escape sequence
base64_data = Base.encode64(resized_image.path)
"\x1b_Ga=T,f=100,s=#{bounds.width},v=#{bounds.height},m=1;#{base64_data}\x1b\\"
else
_ -> "Failed to convert image to kitty format"
end
end
defp create_sixel_cells(sixel_data, bounds) do
case Raxol.Core.ErrorHandling.safe_call(fn ->
create_sixel_cells_from_buffer(sixel_data, bounds)
end) do
{:ok, cells} ->
cells
{:error, reason} ->
Raxol.Core.Runtime.Log.error(
"[ImageRenderer] Error creating Sixel cells: #{inspect(reason)}"
)
empty_cell_grid(bounds.width, bounds.height)
end
end
@doc false
defp create_sixel_cells_from_buffer(sixel_data, %{
width: width,
height: height
}) do
state = Raxol.Terminal.ANSI.SixelGraphics.new()
case Raxol.Terminal.ANSI.SixelGraphics.process_sequence(state, sixel_data) do
{updated_state, :ok} ->
pixel_buffer_to_cells(updated_state, width, height)
{_state, {:error, reason}} ->
Raxol.Core.Runtime.Log.warning_with_context(
"[ImageRenderer] Sixel processing failed: #{inspect(reason)}",
%{}
)
empty_cell_grid(width, height)
end
end
@doc false
@spec pixel_buffer_to_cells(map(), non_neg_integer(), non_neg_integer()) :: [
[Cell.t()]
]
defp pixel_buffer_to_cells(
%{pixel_buffer: buffer, palette: palette},
width,
height
) do
for y <- 0..(height - 1) do
for x <- 0..(width - 1) do
build_cell(buffer, palette, x, y)
end
end
end
@doc false
defp build_cell(buffer, palette, x, y) do
with color_index when not is_nil(color_index) <- Map.get(buffer, {x, y}),
{r, g, b} <- Map.get(palette, color_index) do
Cell.new_sixel(
" ",
%Raxol.Terminal.ANSI.TextFormatting{background: {:rgb, r, g, b}}
)
else
nil -> Cell.new(" ")
_ -> Cell.new_sixel(" ")
end
end
@doc false
@spec empty_cell_grid(non_neg_integer(), non_neg_integer()) :: [[Cell.t()]]
defp empty_cell_grid(width, height) do
List.duplicate(List.duplicate(Cell.new(" "), width), height)
end
defp create_kitty_cells(_kitty_data, %{width: width, height: height}) do
# Kitty graphics protocol support for UI component integration.
# Future enhancement: implement Kitty protocol cell creation
empty_cell_grid(width, height)
end
# --- Private Image Drawing Logic ---
@doc false
# Draws a placeholder box indicating where the image would be.
defp draw_placeholder(_data, title, bounds) do
DrawingUtils.draw_box_with_text(title, bounds)
end
end