Current section

Files

Jump to
bb_kino lib bb kino visualisation.ex
Raw

lib/bb/kino/visualisation.ex

# SPDX-FileCopyrightText: 2025 James Harton
#
# SPDX-License-Identifier: Apache-2.0
defmodule BB.Kino.Visualisation do
@moduledoc """
A Kino widget for 3D robot visualisation using Three.js.
Displays an interactive 3D view of the robot with:
- Real-time joint position updates
- Orbit camera controls (pan, zoom, rotate)
- Visual geometry rendering (boxes, cylinders, spheres)
## Usage
BB.Kino.Visualisation.new(MyRobot)
The widget automatically:
- Loads the robot topology and geometry
- Subscribes to sensor messages for position updates
- Updates joint transforms in real-time
"""
use Kino.JS
use Kino.JS.Live
alias BB.Kino.Shared.RobotContext
alias BB.Robot.Runtime, as: RobotRuntime
alias Kino.JS.Live, as: KinoLive
@doc """
Creates a new 3D visualisation widget for the given robot.
"""
@spec new(module()) :: KinoLive.t()
def new(robot_module) do
KinoLive.new(__MODULE__, robot_module)
end
@impl true
def init(robot_module, ctx) do
case RobotContext.validate_robot(robot_module) do
{:ok, robot} ->
robot_struct = RobotRuntime.get_robot(robot)
positions = RobotRuntime.positions(robot)
BB.subscribe(robot, [:sensor])
{:ok,
assign(ctx,
robot: robot,
robot_struct: robot_struct,
positions: positions
)}
{:error, reason} ->
{:ok, assign(ctx, error: reason)}
end
end
@impl true
def handle_connect(ctx) do
if ctx.assigns[:error] do
{:ok, %{error: ctx.assigns.error}, ctx}
else
topology = serialize_topology(ctx.assigns.robot_struct)
positions = serialize_positions(ctx.assigns.positions)
payload = %{
topology: topology,
positions: positions
}
{:ok, payload, ctx}
end
end
@impl true
def handle_info(
{:bb, [:sensor | _rest], %BB.Message{payload: %BB.Message.Sensor.JointState{} = js}},
ctx
) do
positions =
Enum.zip(js.names, js.positions)
|> Map.new()
position_updates = serialize_positions(positions)
broadcast_event(ctx, "positions_updated", %{positions: position_updates})
{:noreply, assign(ctx, positions: positions)}
end
def handle_info(_msg, ctx) do
{:noreply, ctx}
end
@impl true
def terminate(_reason, ctx) do
if ctx.assigns[:robot] do
BB.unsubscribe(ctx.assigns.robot, [:sensor])
end
:ok
end
# Serialization helpers
defp serialize_topology(robot_struct) do
%{
name: Atom.to_string(robot_struct.name),
links: serialize_links(robot_struct.links),
joints: serialize_joints(robot_struct.joints)
}
end
defp serialize_links(links) do
links
|> Enum.map(fn {name, link} ->
{Atom.to_string(name), serialize_link(link)}
end)
|> Map.new()
end
defp serialize_link(link) do
%{
name: Atom.to_string(link.name),
visual: serialize_visual(link.visual)
}
end
defp serialize_visual(nil), do: nil
defp serialize_visual(visual) do
%{
origin: serialize_origin(visual.origin),
geometry: serialize_geometry(visual.geometry),
material: serialize_material(visual.material)
}
end
defp serialize_origin(nil), do: nil
defp serialize_origin({position, orientation}) do
{px, py, pz} = position
{r, p, y} = orientation
%{
xyz: %{x: px, y: py, z: pz},
rpy: %{r: r, p: p, y: y}
}
end
defp serialize_geometry(nil), do: nil
defp serialize_geometry({:box, dims}) do
%{type: "box", x: dims.x, y: dims.y, z: dims.z}
end
defp serialize_geometry({:cylinder, dims}) do
%{type: "cylinder", radius: dims.radius, length: dims[:height] || dims[:length]}
end
defp serialize_geometry({:sphere, dims}) do
%{type: "sphere", radius: dims.radius}
end
defp serialize_geometry({:mesh, info}) do
%{
type: "mesh",
filename: info.filename,
scale: serialize_scale(info[:scale])
}
end
defp serialize_scale(nil), do: %{x: 1, y: 1, z: 1}
defp serialize_scale(scale) when is_number(scale), do: %{x: scale, y: scale, z: scale}
defp serialize_scale({x, y, z}), do: %{x: x, y: y, z: z}
defp serialize_scale(%{x: x, y: y, z: z}), do: %{x: x, y: y, z: z}
defp serialize_material(nil), do: nil
defp serialize_material(material) do
%{
name: material[:name] && Atom.to_string(material.name),
colour: serialize_colour(material[:color])
}
end
defp serialize_colour(nil), do: nil
defp serialize_colour({r, g, b, a}), do: %{rgba: %{r: r, g: g, b: b, a: a}}
defp serialize_colour({r, g, b}), do: %{r: r, g: g, b: b}
defp serialize_joints(joints) do
joints
|> Enum.map(fn {name, joint} ->
{Atom.to_string(name), serialize_joint(joint)}
end)
|> Map.new()
end
defp serialize_joint(joint) do
%{
name: Atom.to_string(joint.name),
type: Atom.to_string(joint.type),
parent: Atom.to_string(joint.parent_link),
child: Atom.to_string(joint.child_link),
origin: serialize_joint_origin(joint.origin),
axis: serialize_axis(joint.axis),
limits: serialize_limits(joint.limits)
}
end
defp serialize_joint_origin(nil), do: nil
defp serialize_joint_origin(origin) do
{px, py, pz} = origin.position
{r, p, y} = origin.orientation
%{
xyz: %{x: px, y: py, z: pz},
rpy: %{r: r, p: p, y: y}
}
end
defp serialize_axis(nil), do: %{x: 0, y: 0, z: 1}
defp serialize_axis({x, y, z}), do: %{x: x, y: y, z: z}
defp serialize_limits(nil), do: nil
defp serialize_limits(limits) do
%{
lower: limits[:lower],
upper: limits[:upper]
}
end
defp serialize_positions(positions) do
positions
|> Enum.map(fn {name, value} ->
{Atom.to_string(name), value}
end)
|> Map.new()
end
asset "main.js" do
File.read!(Path.join([__DIR__, "..", "..", "..", "priv", "static", "visualisation.js"]))
end
asset "main.css" do
"""
.bb-vis {
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
border: 1px solid #e0e0e0;
border-radius: 8px;
background: #fafafa;
overflow: hidden;
}
.vis-header {
display: flex;
justify-content: space-between;
align-items: center;
padding: 12px 16px;
background: #f5f5f5;
border-bottom: 1px solid #e0e0e0;
}
.title {
font-weight: 600;
font-size: 14px;
color: #333;
}
.vis-controls {
display: flex;
gap: 8px;
}
.vis-controls button {
padding: 6px 12px;
border: none;
border-radius: 4px;
font-size: 12px;
cursor: pointer;
background: #e0e0e0;
color: #333;
}
.vis-controls button:hover {
background: #d0d0d0;
}
.vis-container {
width: 100%;
height: 400px;
background: #f5f5f5;
}
.vis-container canvas {
display: block;
width: 100% !important;
height: 100% !important;
}
.bb-vis-error {
padding: 16px;
}
.error-message {
color: #c62828;
}
"""
end
end