Packages
mob
0.7.20
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.4
0.7.3
0.7.2
0.7.1
0.7.0
0.6.26
0.6.25
0.6.24
0.6.23
0.6.22
0.6.21
0.6.20
0.6.19
0.6.18
0.6.17
0.6.16
0.6.15
0.6.14
0.6.13
0.6.12
0.6.11
0.6.10
0.6.9
0.6.8
0.6.7
0.6.6
0.6.5
0.6.2
0.6.1
0.6.0
0.5.18
0.5.17
0.5.16
0.5.15
0.5.14
0.5.11
0.5.10
0.5.7
0.5.6
0.5.5
0.5.4
0.5.3
0.5.2
0.5.1
0.5.0
0.4.0
0.3.10
0.3.9
0.3.8
0.3.7
0.3.6
0.3.5
0.3.4
0.3.3
0.3.2
0.3.1
0.3.0
0.2.0
0.1.0
BEAM-on-device mobile framework for Elixir
Current section
Files
Jump to
Current section
Files
ios/MobGpuView.swift
// MobGpuView.swift — Metal-backed fragment-shader surface.
//
// Hosts an `MTKView` inside a SwiftUI `UIViewRepresentable`. Compiles the
// MSL fragment shader supplied by the BEAM into a render pipeline, binds
// per-frame uniforms into fragment buffer slot 0, and renders a
// full-screen quad at the display's refresh rate.
//
// Scope (v1):
// - Fragment-shader-only. Built-in vertex shader emits a full-screen
// NDC quad with a (0..1, 0..1) `uv` in `VertexOut.uv`.
// - Uniforms map keys become member names in the `Uniforms` struct.
// Supported types: float (NSNumber), float2/3/4 (NSArray of 2/3/4
// numbers), uint (NSNumber promoted to integer). The uniform struct
// layout is a flat sequence of 16-byte-aligned slots — matches MSL's
// default alignment for vec types.
// - Shader compile errors surface as a translucent red overlay with the
// error message, on top of the (black) Metal view.
//
// Not yet:
// - Textures (camera frame, ML output) as samplers
// - Vertex shader override / custom mesh
// - GLSL → MSL transpilation (escape hatch via the BEAM-side
// %{ios: "..."} map form is the workaround; transpile is a future task)
import Foundation
import Metal
import MetalKit
import SwiftUI
// MARK: - SwiftUI wrapper
struct MobGpuView: UIViewRepresentable {
let node: MobNode
func makeCoordinator() -> Coordinator { Coordinator() }
func makeUIView(context: Context) -> MobGpuMTKView {
let view = MobGpuMTKView(frame: .zero, device: MTLCreateSystemDefaultDevice())
view.backgroundColor = .black
view.colorPixelFormat = .bgra8Unorm
view.framebufferOnly = false
view.preferredFramesPerSecond = 60
view.isPaused = false
view.enableSetNeedsDisplay = false // continuous mode
view.delegate = view // self-delegate; renderer logic lives in MobGpuMTKView
return view
}
func updateUIView(_ view: MobGpuMTKView, context: Context) {
if let shader = node.gpuShaderMSL {
view.setShader(shader)
} else {
view.setShader(nil)
}
view.setUniforms(node.gpuUniforms ?? [])
}
final class Coordinator {}
}
// MARK: - MTKView subclass + renderer
/// A self-delegating MTKView that compiles MSL fragment shaders on demand
/// and renders a full-screen quad with caller-supplied uniforms.
final class MobGpuMTKView: MTKView, MTKViewDelegate {
// Compiled shader pipeline (nil until first valid shader arrives).
private var pipelineState: MTLRenderPipelineState?
private var commandQueue: MTLCommandQueue?
private var compileError: String?
private var currentShaderHash: Int = 0
private var uniformBuffer: MTLBuffer?
private var uniformBytes = Data()
// SwiftUI host for the error overlay. Rendered as a UILabel pinned to
// the top-left so the user sees compile errors inline.
private weak var errorLabel: UILabel?
override init(frame frameRect: CGRect, device: MTLDevice?) {
super.init(frame: frameRect, device: device)
self.commandQueue = device?.makeCommandQueue()
}
required init(coder: NSCoder) {
super.init(coder: coder)
self.commandQueue = self.device?.makeCommandQueue()
}
// MARK: shader handoff from SwiftUI
func setShader(_ source: String?) {
guard let source = source, !source.isEmpty else {
if pipelineState != nil { pipelineState = nil; showError(nil) }
return
}
let hash = source.hashValue
if hash == currentShaderHash, pipelineState != nil { return }
currentShaderHash = hash
compileShader(source)
}
func setUniforms(_ uniforms: Any) {
// Uniforms arrive as a top-level NSArray (BEAM-side list) — packed
// in declaration order so the order survives JSON round-trip and
// map-iteration surprises. Each element is either:
// - NSNumber (float or int → 4-byte slot at natural alignment)
// - NSArray of 2 numbers (float2 → 8-byte slot at 8-byte align)
// - NSArray of 4 numbers (float4 → 16-byte slot at 16-byte align)
//
// The shader then declares its `Uniforms` struct with members in
// the SAME order:
//
// struct Uniforms {
// float2 center; // matches uniforms[0]
// float zoom; // matches uniforms[1]
// uint max_iter; // matches uniforms[2]
// };
//
// (Map form was tempting but Elixir map iteration order is
// not stable beyond ~32 entries and differs across runtimes —
// discovered this empirically when the demo rendered black on
// device because :zoom came first on iOS BEAM.)
var data = Data()
if let list = uniforms as? [Any] {
for value in list {
appendUniformValue(value, to: &data)
}
} else if let dict = uniforms as? [AnyHashable: Any] {
// Fallback for backward compat — iteration order undefined.
// The shader-side struct MUST match whatever the runtime decides.
// Not recommended; use the list form above.
for (_, value) in dict {
appendUniformValue(value, to: &data)
}
}
uniformBytes = data
if data.count > 0 {
uniformBuffer = device?.makeBuffer(bytes: (data as NSData).bytes, length: data.count, options: [])
} else {
uniformBuffer = nil
}
}
private func appendUniformValue(_ value: Any, to data: inout Data) {
if let n = value as? NSNumber {
let typeStr = String(cString: n.objCType)
if typeStr == "q" || typeStr == "l" || typeStr == "i" {
alignTo(4, in: &data)
var v: UInt32 = UInt32(truncatingIfNeeded: n.int64Value)
data.append(Data(bytes: &v, count: 4))
} else {
alignTo(4, in: &data)
var v: Float = n.floatValue
data.append(Data(bytes: &v, count: 4))
}
return
}
if let arr = value as? [Any] {
switch arr.count {
case 2:
alignTo(8, in: &data)
for i in 0..<2 {
if let n = arr[i] as? NSNumber {
var v: Float = n.floatValue
data.append(Data(bytes: &v, count: 4))
}
}
case 4:
alignTo(16, in: &data)
for i in 0..<4 {
if let n = arr[i] as? NSNumber {
var v: Float = n.floatValue
data.append(Data(bytes: &v, count: 4))
}
}
default:
// Unsupported arity (3 reserved for future float3,
// others unhandled). Skip silently — shader-side will
// read garbage, which is at least localizable in a debug.
break
}
}
}
private func alignTo(_ alignment: Int, in data: inout Data) {
let mod = data.count % alignment
if mod != 0 { data.append(Data(count: alignment - mod)) }
}
// MARK: compile
private func compileShader(_ source: String) {
guard let device = device else { return }
let full = """
\(vertexSource)
\(source)
"""
do {
let library = try device.makeLibrary(source: full, options: nil)
guard let vertexFn = library.makeFunction(name: "vertex_main") else {
showError("internal: vertex_main not found in built-in vertex source")
return
}
// Convention: fragment entry point is called `fragment_main`. If
// the supplied shader exports a function with a different name,
// make_function returns nil and we surface that to the user.
guard let fragmentFn = library.makeFunction(name: "fragment_main") else {
showError(
"fragment_main not found — your shader must define " +
"`fragment half4 fragment_main(VertexOut in [[stage_in]], " +
"constant Uniforms& u [[buffer(0)]])`"
)
return
}
let desc = MTLRenderPipelineDescriptor()
desc.vertexFunction = vertexFn
desc.fragmentFunction = fragmentFn
desc.colorAttachments[0].pixelFormat = colorPixelFormat
pipelineState = try device.makeRenderPipelineState(descriptor: desc)
showError(nil)
} catch {
pipelineState = nil
showError(String(describing: error))
}
}
private var vertexSource: String {
// Full-screen quad in clip space + a passthrough uv in (0..1, 0..1).
// The fragment shader writes `Uniforms` member layout itself; we
// don't generate the struct here.
return """
#include <metal_stdlib>
using namespace metal;
struct VertexOut {
float4 position [[position]];
float2 uv;
};
vertex VertexOut vertex_main(uint vid [[vertex_id]]) {
// Quad as a triangle strip: BL, BR, TL, TR
float2 pos[4] = {
float2(-1.0, -1.0),
float2( 1.0, -1.0),
float2(-1.0, 1.0),
float2( 1.0, 1.0)
};
float2 uv[4] = {
float2(0.0, 1.0),
float2(1.0, 1.0),
float2(0.0, 0.0),
float2(1.0, 0.0)
};
VertexOut out;
out.position = float4(pos[vid], 0.0, 1.0);
out.uv = uv[vid];
return out;
}
"""
}
// MARK: error overlay
private func showError(_ message: String?) {
compileError = message
DispatchQueue.main.async { [weak self] in
guard let self = self else { return }
if let message = message {
if self.errorLabel == nil {
let label = UILabel(frame: self.bounds)
label.numberOfLines = 0
label.font = UIFont.monospacedSystemFont(ofSize: 11, weight: .regular)
label.textColor = .white
label.backgroundColor = UIColor.red.withAlphaComponent(0.7)
label.lineBreakMode = .byWordWrapping
label.textAlignment = .left
label.translatesAutoresizingMaskIntoConstraints = false
self.addSubview(label)
NSLayoutConstraint.activate([
label.topAnchor.constraint(equalTo: self.topAnchor),
label.leadingAnchor.constraint(equalTo: self.leadingAnchor),
label.trailingAnchor.constraint(equalTo: self.trailingAnchor)
])
self.errorLabel = label
}
self.errorLabel?.text = "shader error:\n\(message)"
self.errorLabel?.isHidden = false
} else {
self.errorLabel?.isHidden = true
}
}
}
// MARK: MTKViewDelegate
func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {}
func draw(in view: MTKView) {
guard let pipeline = pipelineState,
let cmdBuf = commandQueue?.makeCommandBuffer(),
let renderPass = currentRenderPassDescriptor,
let drawable = currentDrawable,
let encoder = cmdBuf.makeRenderCommandEncoder(descriptor: renderPass)
else {
// Either no shader compiled yet or pipeline failed — let the
// overlay (if any) speak for itself; nothing to draw.
currentDrawable?.present()
return
}
encoder.setRenderPipelineState(pipeline)
if let buf = uniformBuffer {
encoder.setFragmentBuffer(buf, offset: 0, index: 0)
}
encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
encoder.endEncoding()
cmdBuf.present(drawable)
cmdBuf.commit()
}
}