Packages

Revolutionary physics-inspired graph database with 23,742 ops/sec performance

Current section

Files

Jump to
warp_engine lib islab_db wal_entry.ex
Raw

lib/islab_db/wal_entry.ex

defmodule IsLabDB.WAL.Entry do
@moduledoc """
WAL Entry structure for high-performance operation logging.
Defines the format for Write-Ahead Log entries that capture all
database operations while maintaining physics intelligence metadata.
## Design Principles
- **Hybrid Format**: Binary for speed, JSON for debugging
- **Physics Preservation**: Complete cosmic metadata included
- **Compression**: Large values compressed automatically
- **Versioning**: Forward/backward compatibility
## Entry Structure
Each WAL entry contains:
- Sequence number for ordering
- High-precision timestamp
- Operation type (:put, :get, :delete)
- Spacetime shard information
- Data key and value
- Complete physics metadata
"""
# Jason.Encoder derivation removed for compatibility - will be added back when Jason is available
defstruct [
:sequence, # Unique sequence number for ordering
:timestamp, # Microsecond timestamp
:operation, # :put, :get, :delete, :quantum_get
:shard_id, # Spacetime shard identifier
:key, # Data key
:value, # Data value (may be compressed)
:value_preview, # Small preview for JSON debugging
:physics_metadata, # Complete cosmic metadata
:compression_type, # :none, :gzip, :lz4
:version, # Entry format version
:checksum # Data integrity verification
]
# Entry format version for compatibility
@current_version "6.6.0"
# Compression thresholds
@compression_threshold_bytes 1024
@preview_max_length 100
@doc """
Create a new WAL entry from operation parameters.
"""
def new(operation_type, key, value, shard_id, physics_metadata, sequence) do
timestamp = :os.system_time(:microsecond)
# Compress large values for efficiency
{final_value, compression_type} = compress_if_large(value)
# Create value preview for JSON debugging
value_preview = create_value_preview(value)
# Calculate checksum for integrity
checksum = calculate_checksum(key, final_value, physics_metadata)
entry = %__MODULE__{
sequence: sequence,
timestamp: timestamp,
operation: operation_type,
shard_id: shard_id,
key: key,
value: final_value,
value_preview: value_preview,
physics_metadata: physics_metadata,
compression_type: compression_type,
version: @current_version,
checksum: checksum
}
entry
end
@doc """
Encode entry to binary format for maximum I/O performance.
Binary format structure:
- Version (8 bytes)
- Sequence (8 bytes)
- Timestamp (8 bytes)
- Operation type (1 byte)
- Compression type (1 byte)
- Key length + key data
- Value length + value data
- Physics metadata length + metadata
- Checksum (16 bytes)
"""
def encode_binary(entry) do
key_binary = :erlang.term_to_binary(entry.key)
value_binary = entry.value
metadata_binary = :erlang.term_to_binary(entry.physics_metadata)
operation_byte = operation_to_byte(entry.operation)
compression_byte = compression_to_byte(entry.compression_type)
# Build binary entry
<<
entry.sequence::64,
entry.timestamp::64,
operation_byte::8,
compression_byte::8,
byte_size(key_binary)::32,
key_binary::binary,
byte_size(value_binary)::32,
value_binary::binary,
byte_size(metadata_binary)::32,
metadata_binary::binary,
entry.checksum::128
>>
end
@doc """
Decode entry from binary format.
"""
def decode_binary(binary_data) do
<<
sequence::64,
timestamp::64,
operation_byte::8,
compression_byte::8,
key_size::32,
key_binary::binary-size(key_size),
value_size::32,
value_binary::binary-size(value_size),
metadata_size::32,
metadata_binary::binary-size(metadata_size),
checksum::128,
_rest::binary
>> = binary_data
key = :erlang.binary_to_term(key_binary)
physics_metadata = :erlang.binary_to_term(metadata_binary)
operation = byte_to_operation(operation_byte)
compression_type = byte_to_compression(compression_byte)
# Decompress value if needed
final_value = decompress_value(value_binary, compression_type)
%__MODULE__{
sequence: sequence,
timestamp: timestamp,
operation: operation,
key: key,
value: final_value,
physics_metadata: physics_metadata,
compression_type: compression_type,
version: @current_version,
checksum: checksum
}
end
@doc """
Encode entry to JSON format for human-readable debugging.
"""
def encode_json(entry) do
# Manual JSON encoding for compatibility
json_map = %{
sequence: entry.sequence,
timestamp: entry.timestamp,
operation: entry.operation,
shard_id: entry.shard_id,
key: entry.key,
value_preview: entry.value_preview,
physics_metadata: entry.physics_metadata,
compression_type: entry.compression_type,
version: entry.version
}
# Simple JSON encoding (will be replaced with proper Jason when available)
inspect(json_map, pretty: true)
end
@doc """
Decode entry from JSON format.
"""
def decode_json(_json_data) do
# JSON decoding not implemented without Jason
{:error, "JSON decoding requires Jason library"}
end
@doc """
Validate entry integrity using checksum.
"""
def validate_integrity(entry) do
expected_checksum = calculate_checksum(entry.key, entry.value, entry.physics_metadata)
entry.checksum == expected_checksum
end
@doc """
Get entry size in bytes (for statistics and monitoring).
"""
def entry_byte_size(entry) do
key_size = :erlang.size(:erlang.term_to_binary(entry.key))
value_size = if is_binary(entry.value), do: byte_size(entry.value), else: :erlang.size(:erlang.term_to_binary(entry.value))
metadata_size = :erlang.size(:erlang.term_to_binary(entry.physics_metadata))
# Header overhead (sequence, timestamp, operation, etc.)
header_size = 64
header_size + key_size + value_size + metadata_size
end
## PRIVATE FUNCTIONS
defp compress_if_large(value) do
value_binary = if is_binary(value), do: value, else: :erlang.term_to_binary(value)
if byte_size(value_binary) > @compression_threshold_bytes do
compressed = :zlib.gzip(value_binary)
{compressed, :gzip}
else
{value_binary, :none}
end
end
defp decompress_value(value_binary, :gzip) do
decompressed = :zlib.gunzip(value_binary)
# Try to decode as Erlang term, otherwise keep as binary
try do
:erlang.binary_to_term(decompressed)
catch
_, _ -> decompressed
end
end
defp decompress_value(value_binary, :none) do
# Try to decode as Erlang term, otherwise keep as binary
try do
:erlang.binary_to_term(value_binary)
catch
_, _ -> value_binary
end
end
defp create_value_preview(value) do
preview = inspect(value, limit: @preview_max_length, printable_limit: @preview_max_length)
if String.length(preview) > @preview_max_length do
String.slice(preview, 0, @preview_max_length) <> "..."
else
preview
end
end
defp calculate_checksum(key, value, physics_metadata) do
# Create hash of critical data for integrity verification
data_to_hash = [
:erlang.term_to_binary(key),
if(is_binary(value), do: value, else: :erlang.term_to_binary(value)),
:erlang.term_to_binary(physics_metadata)
]
combined_data = Enum.join(data_to_hash)
:crypto.hash(:md5, combined_data)
|> :binary.decode_unsigned()
end
defp operation_to_byte(:put), do: 1
defp operation_to_byte(:get), do: 2
defp operation_to_byte(:delete), do: 3
defp operation_to_byte(:quantum_get), do: 4
defp operation_to_byte(_), do: 0
defp byte_to_operation(1), do: :put
defp byte_to_operation(2), do: :get
defp byte_to_operation(3), do: :delete
defp byte_to_operation(4), do: :quantum_get
defp byte_to_operation(_), do: :unknown
defp compression_to_byte(:none), do: 0
defp compression_to_byte(:gzip), do: 1
defp compression_to_byte(:lz4), do: 2
defp compression_to_byte(_), do: 0
defp byte_to_compression(0), do: :none
defp byte_to_compression(1), do: :gzip
defp byte_to_compression(2), do: :lz4
defp byte_to_compression(_), do: :none
defp atomize_keys(map) when is_map(map) do
Map.new(map, fn {k, v} -> {String.to_atom(k), v} end)
end
defp atomize_keys(value), do: value
end