Packages
macula
0.20.21
7.1.0
7.0.0
6.0.0
5.2.2
5.2.1
5.2.0
5.1.0
5.0.0
4.8.0
4.7.1
4.7.0
4.6.0
4.5.0
4.4.10
4.4.9
4.4.8
4.4.7
4.4.6
4.4.5
4.4.4
4.4.3
4.4.2
4.4.1
4.4.0
4.3.1
4.3.0
4.2.9
4.2.8
4.2.7
4.2.6
4.2.5
4.2.4
4.2.3
4.2.2
4.2.1
4.2.0
4.1.1
4.1.0
4.0.0
3.16.0
3.15.3
3.15.2
3.15.1
3.14.0
3.13.0
3.12.1
3.12.0
3.11.1
3.11.0
3.10.3
3.10.2
3.10.1
3.9.0
3.8.0
3.7.0
3.5.0
3.4.0
3.3.0
3.2.0
3.1.0
3.0.0
2.1.1
2.1.0
2.0.0
1.5.2
1.5.1
1.4.30
1.4.29
1.4.28
1.4.27
1.4.26
1.4.25
1.4.24
1.4.23
1.4.22
1.4.21
1.4.20
1.4.19
1.4.18
1.4.17
1.4.16
1.4.15
1.4.14
1.4.13
1.4.11
1.4.10
1.4.9
1.4.8
1.4.7
1.4.6
1.4.5
1.4.4
1.4.3
1.4.2
1.4.1
1.4.0
1.3.1
1.3.0
1.2.0
1.1.0
1.0.10
1.0.9
1.0.8
1.0.7
1.0.6
1.0.5
1.0.4
1.0.3
1.0.2
1.0.1
1.0.0
0.48.6
0.48.5
0.48.4
0.48.3
0.48.2
0.48.1
0.48.0
0.47.1
0.47.0
0.46.3
0.46.1
0.46.0
0.45.3
0.45.2
0.45.1
0.45.0
0.44.2
0.44.1
0.44.0
0.43.3
0.43.2
0.43.1
0.43.0
0.42.9
0.42.8
0.42.7
0.42.6
0.42.5
0.42.4
0.42.3
0.42.2
0.42.1
0.42.0
0.41.1
0.41.0
0.40.1
0.40.0
0.39.9
0.39.8
0.39.7
0.39.6
0.39.5
0.39.4
0.39.3
0.39.2
0.39.1
0.39.0
0.38.8
0.38.7
0.38.6
0.38.5
0.38.4
0.38.3
0.38.2
0.38.1
0.38.0
0.37.7
0.37.6
0.37.5
0.37.4
0.37.3
0.37.2
0.37.1
0.37.0
0.36.6
0.36.5
0.36.4
0.36.3
0.36.2
0.36.1
0.36.0
0.35.4
0.35.3
0.35.2
0.35.1
0.35.0
0.34.1
0.34.0
0.33.1
0.33.0
0.32.5
0.32.4
0.32.3
0.32.2
0.32.1
0.32.0
0.31.9
0.31.8
0.31.7
0.31.6
0.31.5
0.31.4
0.31.3
0.31.2
0.31.1
0.31.0
0.30.10
0.30.9
0.30.8
0.30.7
0.30.6
0.30.5
0.30.4
0.30.3
0.30.2
0.30.1
0.30.0
0.29.0
0.28.3
0.28.2
0.28.1
0.28.0
0.27.1
0.27.0
0.26.1
0.26.0
0.25.6
0.25.5
0.25.4
0.25.3
0.25.2
0.25.1
0.25.0
0.24.6
0.24.5
0.24.4
0.24.3
0.24.2
0.24.1
0.24.0
0.23.3
0.23.2
0.23.1
0.23.0
0.22.12
0.22.11
0.22.10
0.22.9
0.22.8
0.22.7
0.22.6
0.22.5
0.22.4
0.22.3
0.22.2
0.22.1
0.22.0
0.21.7
0.21.6
0.21.5
0.21.4
0.21.2
0.21.1
0.21.0
0.20.25
0.20.24
0.20.23
0.20.22
0.20.21
0.20.20
0.20.19
0.20.18
0.20.17
0.20.16
0.20.15
0.20.14
0.20.13
0.20.12
0.20.11
0.20.10
0.20.9
0.20.8
0.20.7
0.20.6
0.20.5
0.20.3
0.20.2
0.20.1
0.20.0
0.19.2
0.19.1
0.19.0
0.18.1
0.18.0
0.17.4
0.17.3
0.17.2
0.17.1
0.17.0
0.16.6
0.16.5
0.16.4
0.16.3
0.16.2
0.16.1
0.16.0
0.15.1
0.15.0
0.14.3
0.14.2
0.14.1
0.14.0
0.12.6
0.12.5
0.12.3
0.11.3
0.10.2
0.10.1
0.10.0
0.9.2
0.9.1
0.9.0
0.8.25
0.8.24
0.8.23
0.8.22
0.8.21
0.8.20
0.8.19
0.8.18
0.8.17
0.8.16
0.8.15
0.8.14
0.8.13
0.8.12
0.8.11
0.8.10
0.8.9
0.8.8
0.8.7
0.8.6
0.8.5
0.8.4
0.8.3
0.8.2
0.8.1
0.8.0
0.7.30
0.7.29
0.7.28
0.7.27
0.7.26
0.7.25
0.7.24
0.7.23
0.7.22
0.7.21
0.7.20
0.7.19
0.7.18
0.7.17
0.7.16
0.7.15
0.7.14
0.7.13
0.7.12
0.7.11
0.7.10
0.7.9
0.7.8
0.7.7
0.7.6
0.7.5
0.7.4
0.7.3
0.7.2
0.7.1
0.7.0
0.6.7
0.6.6
0.6.5
0.6.4
0.6.3
0.6.2
0.6.1
0.6.0
0.5.0
0.4.4
0.4.3
0.4.2
0.4.1
0.4.0
0.3.4
0.3.3
0.3.2
0.3.1
Macula HTTP/3 Mesh SDK — connect, subscribe, publish, call, advertise
Current section
Files
Jump to
Current section
Files
src/macula_content_system/macula_content_manifest.erl
%%%-------------------------------------------------------------------
%%% @doc
%%% Content manifest module for Macula content-addressed storage.
%%%
%%% Manifests describe content: name, size, chunk information, and
%%% Merkle root hash for verification. They are identified by an MCID
%%% (Macula Content Identifier) which is a self-describing hash.
%%%
%%% == MCID Format ==
%%% MCIDs are 34-byte binary values:
%%% - Byte 0: Version (currently 0x01)
%%% - Byte 1: Codec (0x55=raw, 0x56=manifest)
%%% - Bytes 2-33: 32-byte hash (BLAKE3 or SHA256)
%%%
%%% == String Representation ==
%%% MCIDs can be converted to human-readable strings:
%%% `mcid1-manifest-blake3-5d41402abc4b2a76b9719d911017c592'
%%%
%%% == Example Usage ==
%%% ```
%%% %% Create manifest from binary data
%%% {ok, Manifest} = macula_content_manifest:create(Data, #{
%%% name => <<"my_app-1.0.0.tar.gz"/utf8>>,
%%% chunk_size => 262144
%%% }).
%%%
%%% %% Get MCID as string
%%% MCIDString = macula_content_manifest:mcid_to_string(maps:get(mcid, Manifest)).
%%%
%%% %% Verify data matches manifest
%%% ok = macula_content_manifest:verify(Manifest, Data).
%%% '''
%%% @end
%%%-------------------------------------------------------------------
-module(macula_content_manifest).
%% API
-export([
create/2,
encode/1,
decode/1,
verify/2,
mcid_to_string/1,
mcid_from_string/1,
get_chunk_mcid/2,
version/0
]).
%% Types
-type mcid() :: <<_:272>>. %% 34 bytes
-type manifest() :: #{
mcid := mcid(),
version := pos_integer(),
name := binary(),
size := non_neg_integer(),
created := pos_integer(),
chunk_size := pos_integer(),
chunk_count := non_neg_integer(),
hash_algorithm := macula_content_hasher:algorithm(),
root_hash := binary(),
chunks := [macula_content_chunker:chunk_info()],
signature => binary(),
publisher_did => binary()
}.
-export_type([mcid/0, manifest/0]).
%% Constants
-define(VERSION, 1).
-define(CODEC_RAW, 16#55).
-define(CODEC_MANIFEST, 16#56).
%%%===================================================================
%%% API Functions
%%%===================================================================
%% @doc Return the current manifest version.
-spec version() -> pos_integer().
version() ->
?VERSION.
%% @doc Create a manifest from binary data.
%%
%% Options:
%% <ul>
%% <li>`name' - binary() - Content name (default: "unnamed")</li>
%% <li>`chunk_size' - pos_integer() - Chunk size (default: 262144)</li>
%% <li>`hash_algorithm' - blake3 | sha256 - Hash algorithm (default: blake3)</li>
%% </ul>
-spec create(binary(), map()) -> {ok, manifest()}.
create(Data, Opts) ->
Name = maps:get(name, Opts, <<"unnamed">>),
ChunkSize = maps:get(chunk_size, Opts, macula_content_chunker:default_chunk_size()),
Algorithm = maps:get(hash_algorithm, Opts, blake3),
%% Chunk the data
{ok, Chunks} = macula_content_chunker:chunk(Data, ChunkSize),
ChunkInfos = macula_content_chunker:chunk_info(Chunks, Algorithm),
%% Compute Merkle root
RootHash = case ChunkInfos of
[] -> macula_content_hasher:hash(Algorithm, <<>>);
_ -> macula_content_chunker:merkle_root(ChunkInfos, Algorithm)
end,
%% Build manifest map (without MCID first)
ManifestData = #{
version => ?VERSION,
name => Name,
size => byte_size(Data),
created => erlang:system_time(second),
chunk_size => ChunkSize,
chunk_count => length(ChunkInfos),
hash_algorithm => Algorithm,
root_hash => RootHash,
chunks => ChunkInfos
},
%% Compute MCID from manifest content
MCID = compute_mcid(ManifestData, Algorithm),
{ok, ManifestData#{mcid => MCID}}.
%% @doc Encode manifest to binary (MessagePack).
-spec encode(manifest()) -> {ok, binary()} | {error, term()}.
encode(Manifest) ->
%% Convert to encodable format
Encodable = manifest_to_encodable(Manifest),
case msgpack:pack(Encodable, [{map_format, map}]) of
{error, Reason} -> {error, Reason};
Packed -> {ok, Packed}
end.
%% @doc Decode manifest from binary.
-spec decode(binary()) -> {ok, manifest()} | {error, invalid_manifest}.
decode(Binary) ->
case msgpack:unpack(Binary, [{map_format, map}]) of
{ok, Map} -> decode_manifest(Map);
{error, _} -> {error, invalid_manifest}
end.
%% @doc Verify that data matches the manifest.
-spec verify(manifest(), binary()) -> ok | {error, size_mismatch | root_hash_mismatch}.
verify(Manifest, Data) ->
Size = maps:get(size, Manifest),
case byte_size(Data) of
Size ->
verify_root_hash(Manifest, Data);
_ ->
{error, size_mismatch}
end.
%% @doc Convert MCID to human-readable string.
-spec mcid_to_string(mcid()) -> binary().
mcid_to_string(<<Version:8, Codec:8, Hash:32/binary>>) ->
VersionStr = integer_to_binary(Version),
CodecStr = codec_to_string(Codec),
AlgoStr = <<"blake3">>, %% Assume blake3 for now
HashHex = macula_content_hasher:hex_encode(Hash),
<<"mcid", VersionStr/binary, "-", CodecStr/binary, "-",
AlgoStr/binary, "-", HashHex/binary>>.
%% @doc Parse MCID from string.
-spec mcid_from_string(binary()) -> {ok, mcid()} | {error, invalid_mcid}.
mcid_from_string(String) ->
case parse_mcid_string(String) of
{ok, Version, Codec, Hash} ->
{ok, <<Version:8, Codec:8, Hash/binary>>};
error ->
{error, invalid_mcid}
end.
%% @doc Get MCID for a specific chunk.
-spec get_chunk_mcid(manifest(), non_neg_integer()) -> {ok, mcid()} | {error, invalid_index}.
get_chunk_mcid(Manifest, Index) ->
Chunks = maps:get(chunks, Manifest),
case Index < length(Chunks) of
true ->
ChunkInfo = lists:nth(Index + 1, Chunks),
Hash = maps:get(hash, ChunkInfo),
MCID = make_mcid(?VERSION, ?CODEC_RAW, Hash),
{ok, MCID};
false ->
{error, invalid_index}
end.
%%%===================================================================
%%% Internal Functions
%%%===================================================================
%% @private
compute_mcid(ManifestData, Algorithm) ->
%% Create deterministic encoding of manifest (sorted keys, no timestamps for MCID)
Canonical = #{
<<"name">> => maps:get(name, ManifestData),
<<"size">> => maps:get(size, ManifestData),
<<"chunk_size">> => maps:get(chunk_size, ManifestData),
<<"chunk_count">> => maps:get(chunk_count, ManifestData),
<<"hash_algorithm">> => atom_to_binary(maps:get(hash_algorithm, ManifestData)),
<<"root_hash">> => maps:get(root_hash, ManifestData)
},
Packed = msgpack:pack(Canonical, [{map_format, map}]),
Hash = macula_content_hasher:hash(Algorithm, Packed),
make_mcid(?VERSION, ?CODEC_MANIFEST, Hash).
%% @private
make_mcid(Version, Codec, Hash) ->
<<Version:8, Codec:8, Hash/binary>>.
%% @private
codec_to_string(?CODEC_RAW) -> <<"raw">>;
codec_to_string(?CODEC_MANIFEST) -> <<"manifest">>;
codec_to_string(_) -> <<"unknown">>.
%% @private
string_to_codec(<<"raw">>) -> {ok, ?CODEC_RAW};
string_to_codec(<<"manifest">>) -> {ok, ?CODEC_MANIFEST};
string_to_codec(_) -> error.
%% @private
parse_mcid_string(<<"mcid", Rest/binary>>) ->
case binary:split(Rest, <<"-">>, [global]) of
[VersionStr, CodecStr, _AlgoStr, HashHex] ->
Version = binary_to_integer(VersionStr),
case string_to_codec(CodecStr) of
{ok, Codec} ->
case macula_content_hasher:hex_decode(HashHex) of
{ok, Hash} when byte_size(Hash) =:= 32 ->
{ok, Version, Codec, Hash};
_ ->
error
end;
error ->
error
end;
_ ->
error
end;
parse_mcid_string(_) ->
error.
%% @private
manifest_to_encodable(Manifest) ->
#{
<<"mcid">> => maps:get(mcid, Manifest),
<<"version">> => maps:get(version, Manifest),
<<"name">> => maps:get(name, Manifest),
<<"size">> => maps:get(size, Manifest),
<<"created">> => maps:get(created, Manifest),
<<"chunk_size">> => maps:get(chunk_size, Manifest),
<<"chunk_count">> => maps:get(chunk_count, Manifest),
<<"hash_algorithm">> => atom_to_binary(maps:get(hash_algorithm, Manifest)),
<<"root_hash">> => maps:get(root_hash, Manifest),
<<"chunks">> => [chunk_info_to_encodable(C) || C <- maps:get(chunks, Manifest)]
}.
%% @private
chunk_info_to_encodable(ChunkInfo) ->
#{
<<"index">> => maps:get(index, ChunkInfo),
<<"offset">> => maps:get(offset, ChunkInfo),
<<"size">> => maps:get(size, ChunkInfo),
<<"hash">> => maps:get(hash, ChunkInfo)
}.
%% @private
decode_manifest(Map) when is_map(Map) ->
case maps:is_key(<<"mcid">>, Map) of
true ->
{ok, #{
mcid => maps:get(<<"mcid">>, Map),
version => maps:get(<<"version">>, Map, 1),
name => maps:get(<<"name">>, Map, <<"unnamed">>),
size => maps:get(<<"size">>, Map, 0),
created => maps:get(<<"created">>, Map, 0),
chunk_size => maps:get(<<"chunk_size">>, Map, 262144),
chunk_count => maps:get(<<"chunk_count">>, Map, 0),
hash_algorithm => binary_to_existing_atom(
maps:get(<<"hash_algorithm">>, Map, <<"blake3">>), utf8),
root_hash => maps:get(<<"root_hash">>, Map, <<>>),
chunks => [decode_chunk_info(C) || C <- maps:get(<<"chunks">>, Map, [])]
}};
false ->
{error, invalid_manifest}
end;
decode_manifest(_) ->
{error, invalid_manifest}.
%% @private
decode_chunk_info(Map) ->
#{
index => maps:get(<<"index">>, Map, 0),
offset => maps:get(<<"offset">>, Map, 0),
size => maps:get(<<"size">>, Map, 0),
hash => maps:get(<<"hash">>, Map, <<>>)
}.
%% @private
verify_root_hash(Manifest, Data) ->
ChunkSize = maps:get(chunk_size, Manifest),
Algorithm = maps:get(hash_algorithm, Manifest),
ExpectedRoot = maps:get(root_hash, Manifest),
%% Rechunk and compute root
{ok, Chunks} = macula_content_chunker:chunk(Data, ChunkSize),
ChunkInfos = macula_content_chunker:chunk_info(Chunks, Algorithm),
ActualRoot = case ChunkInfos of
[] -> macula_content_hasher:hash(Algorithm, <<>>);
_ -> macula_content_chunker:merkle_root(ChunkInfos, Algorithm)
end,
case ActualRoot =:= ExpectedRoot of
true -> ok;
false -> {error, root_hash_mismatch}
end.