Packages

crucible_model_registry

0.3.0
0.3.1 0.3.0 0.2.0 0.1.0

Model versioning, artifact storage, and lineage tracking for ML pipelines

HexDocs

Current section

4 Versions

Jump to
Readme
4 Versions
11 Dependencies
1 Dependant
Files
Activity
Hide whitespace

Compare versions

8 files changed
+142 additions
-34 deletions
  @@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file.
5 5 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6 6 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7 7
8 + ## [0.3.0] - 2025-12-28
9 +
10 + ### Changed
11 +
12 + - **Breaking**: Adopted Oban-style dynamic repo injection - host applications now provide their own Repo via `config :crucible_model_registry, repo: MyApp.Repo`
13 + - Repo is no longer auto-started; host app manages its own supervision tree
14 + - Added `CrucibleModelRegistry.repo/0` function for accessing configured repo
15 + - Relaxed Elixir version requirement to ~> 1.14
16 + - Updated postgrex to >= 0.21.1
17 + - Updated telemetry to ~> 1.3
18 + - Updated crucible_framework to ~> 0.5.1
19 +
20 + ### Added
21 +
22 + - Legacy mode support via `start_repo: true` config for backwards compatibility
23 + - `mix crucible_model_registry.install` task documentation for migrations
24 +
8 25 ## [0.2.0] - 2025-12-27
9 26
10 27 ### Added
  @@ -38,17 +38,56 @@ end
38 38
39 39 ## Configuration
40 40
41 + ### Database Configuration (Oban Pattern)
42 +
43 + CrucibleModelRegistry uses dynamic repo injection - your host application provides the Repo:
44 +
41 45 ```elixir
42 46 import Config
43 47
44 48 config :crucible_model_registry,
45 - ecto_repos: [CrucibleModelRegistry.Repo],
49 + repo: MyApp.Repo, # Required: host app's Repo module
46 50 storage_backend: :s3,
47 51 storage_opts: [
48 52 bucket: "my-model-artifacts",
49 53 region: "us-east-1"
50 54 ]
51 55
56 + # Your host app's Repo configuration
57 + config :my_app, MyApp.Repo,
58 + database: "my_app_dev",
59 + username: "postgres",
60 + password: "postgres",
61 + hostname: "localhost"
62 + ```
63 +
64 + Then start your Repo in your application's supervision tree:
65 +
66 + ```elixir
67 + # lib/my_app/application.ex
68 + children = [
69 + MyApp.Repo,
70 + # ... other children
71 + ]
72 + ```
73 +
74 + ### Migrations
75 +
76 + Copy migrations from `deps/crucible_model_registry/priv/repo/migrations/` or run:
77 +
78 + ```bash
79 + mix crucible_model_registry.install
80 + ```
81 +
82 + ### Legacy Mode
83 +
84 + For backwards compatibility, set `start_repo: true` to auto-start internal Repo:
85 +
86 + ```elixir
87 + config :crucible_model_registry,
88 + start_repo: true,
89 + ecto_repos: [CrucibleModelRegistry.Repo]
90 +
52 91 config :crucible_model_registry, CrucibleModelRegistry.Repo,
53 92 database: "crucible_model_registry_dev",
54 93 username: "postgres",
  @@ -4,10 +4,10 @@
4 4 {<<"Online documentation">>,
5 5 <<"https://hexdocs.pm/crucible_model_registry">>}]}.
6 6 {<<"name">>,<<"crucible_model_registry">>}.
7 - {<<"version">>,<<"0.2.0">>}.
7 + {<<"version">>,<<"0.3.0">>}.
8 8 {<<"description">>,
9 9 <<"Model versioning, artifact storage, and lineage tracking for ML pipelines">>}.
10 - {<<"elixir">>,<<"~> 1.18">>}.
10 + {<<"elixir">>,<<"~> 1.14">>}.
11 11 {<<"app">>,<<"crucible_model_registry">>}.
12 12 {<<"files">>,
13 13 [<<"README.md">>,<<"CHANGELOG.md">>,<<"mix.exs">>,<<"LICENSE">>,<<"lib">>,
  @@ -53,7 +53,7 @@
53 53 [{<<"name">>,<<"postgrex">>},
54 54 {<<"app">>,<<"postgrex">>},
55 55 {<<"optional">>,false},
56 - {<<"requirement">>,<<"~> 0.18">>},
56 + {<<"requirement">>,<<">= 0.21.1">>},
57 57 {<<"repository">>,<<"hexpm">>}],
58 58 [{<<"name">>,<<"jason">>},
59 59 {<<"app">>,<<"jason">>},
  @@ -63,7 +63,7 @@
63 63 [{<<"name">>,<<"telemetry">>},
64 64 {<<"app">>,<<"telemetry">>},
65 65 {<<"optional">>,false},
66 - {<<"requirement">>,<<"~> 1.2">>},
66 + {<<"requirement">>,<<"~> 1.3">>},
67 67 {<<"repository">>,<<"hexpm">>}],
68 68 [{<<"name">>,<<"libgraph">>},
69 69 {<<"app">>,<<"libgraph">>},
  @@ -93,7 +93,7 @@
93 93 [{<<"name">>,<<"crucible_framework">>},
94 94 {<<"app">>,<<"crucible_framework">>},
95 95 {<<"optional">>,false},
96 - {<<"requirement">>,<<"~> 0.5.0">>},
96 + {<<"requirement">>,<<"~> 0.5.1">>},
97 97 {<<"repository">>,<<"hexpm">>}],
98 98 [{<<"name">>,<<"crucible_ir">>},
99 99 {<<"app">>,<<"crucible_ir">>},
  @@ -1,11 +1,42 @@
1 1 defmodule CrucibleModelRegistry do
2 2 @moduledoc """
3 3 Model registry for ML artifacts with lineage tracking.
4 +
5 + ## Database Configuration
6 +
7 + CrucibleModelRegistry requires a Repo for persistence. Configure it in your host application:
8 +
9 + config :crucible_model_registry, repo: MyApp.Repo
10 +
11 + Then start your Repo in your supervision tree. Run migrations:
12 +
13 + mix crucible_model_registry.install
14 +
15 + Or copy migrations from `deps/crucible_model_registry/priv/repo/migrations/`.
4 16 """
5 17
6 - alias CrucibleModelRegistry.{ConfigHash, Lineage, Query, Repo, Storage}
18 + alias CrucibleModelRegistry.{ConfigHash, Lineage, Query, Storage}
7 19 alias CrucibleModelRegistry.Schemas.{Artifact, Model, ModelVersion}
8 20
21 + @doc """
22 + Returns the configured Repo module.
23 +
24 + Raises if not configured. Configure with:
25 +
26 + config :crucible_model_registry, repo: MyApp.Repo
27 + """
28 + @spec repo() :: module()
29 + def repo do
30 + Application.get_env(:crucible_model_registry, :repo) ||
31 + raise ArgumentError, """
32 + CrucibleModelRegistry requires a :repo configuration.
33 +
34 + Add to your config:
35 +
36 + config :crucible_model_registry, repo: MyApp.Repo
37 + """
38 + end
39 +
9 40 @type stage :: ModelVersion.stage()
10 41 @type artifact_type :: Artifact.artifact_type()
11 42
  @@ -15,7 +46,7 @@ defmodule CrucibleModelRegistry do
15 46 with {:ok, attrs} <- normalize_register_params(params),
16 47 :ok <- ensure_unique_config(attrs.config_hash) do
17 48 result =
18 - Repo.transaction(fn ->
49 + repo().transaction(fn ->
19 50 model = get_or_create_model!(attrs.model_name, Map.get(attrs, :model_meta, %{}))
20 51
21 52 {:ok, version} =
  @@ -32,14 +63,14 @@ defmodule CrucibleModelRegistry do
32 63 parent_version_id: attrs.parent_version_id,
33 64 lineage_type: attrs.lineage_type
34 65 })
35 - |> Repo.insert()
66 + |> repo().insert()
36 67
37 68 artifacts = insert_artifacts!(version, attrs.artifacts)
38 69
39 70 case maybe_insert_lineage_edge!(version, attrs.parent_version_id, attrs.lineage_type) do
40 71 :ok -> :ok
41 72 {:ok, _edge} -> :ok
42 - {:error, reason} -> Repo.rollback(reason)
73 + {:error, reason} -> repo().rollback(reason)
43 74 end
44 75
45 76 version = %{version | artifacts: artifacts, model: model}
  @@ -66,7 +97,7 @@ defmodule CrucibleModelRegistry do
66 97 @doc "Get a model by name."
67 98 @spec get_model(String.t()) :: {:ok, Model.t()} | {:error, :not_found}
68 99 def get_model(name) do
69 - case Repo.get_by(Model, name: name) do
100 + case repo().get_by(Model, name: name) do
70 101 nil -> {:error, :not_found}
71 102 model -> {:ok, model}
72 103 end
  @@ -75,7 +106,7 @@ defmodule CrucibleModelRegistry do
75 106 @doc "Get a model version by id."
76 107 @spec get_version(integer()) :: {:ok, ModelVersion.t()} | {:error, :not_found}
77 108 def get_version(id) do
78 - case Repo.get(ModelVersion, id) do
109 + case repo().get(ModelVersion, id) do
79 110 nil -> {:error, :not_found}
80 111 version -> {:ok, version}
81 112 end
  @@ -96,7 +127,7 @@ defmodule CrucibleModelRegistry do
96 127 @doc "Find a version by config hash."
97 128 @spec find_by_config_hash(String.t()) :: {:ok, ModelVersion.t()} | :not_found
98 129 def find_by_config_hash(hash) when is_binary(hash) do
99 - case Repo.get_by(ModelVersion, config_hash: hash) do
130 + case repo().get_by(ModelVersion, config_hash: hash) do
100 131 nil -> :not_found
101 132 version -> {:ok, version}
102 133 end
  @@ -107,7 +138,7 @@ defmodule CrucibleModelRegistry do
107 138 def promote(%ModelVersion{} = version, new_stage) do
108 139 version
109 140 |> ModelVersion.changeset(%{stage: new_stage})
110 - |> Repo.update()
141 + |> repo().update()
111 142 |> case do
112 143 {:ok, updated} ->
113 144 emit([:promote], %{count: 1}, %{id: updated.id, stage: updated.stage})
  @@ -122,7 +153,7 @@ defmodule CrucibleModelRegistry do
122 153 @spec upload_artifact(ModelVersion.t(), artifact_type(), Path.t()) ::
123 154 {:ok, Artifact.t()} | {:error, term()}
124 155 def upload_artifact(%ModelVersion{} = version, type, local_path) do
125 - version = Repo.preload(version, :model)
156 + version = repo().preload(version, :model)
126 157 remote_path = artifact_remote_path(version.model.name, version.version, type, local_path)
127 158
128 159 case Storage.upload(local_path, remote_path, []) do
  @@ -138,7 +169,7 @@ defmodule CrucibleModelRegistry do
138 169
139 170 %Artifact{}
140 171 |> Artifact.changeset(attrs)
141 - |> Repo.insert()
172 + |> repo().insert()
142 173 |> case do
143 174 {:ok, artifact} ->
144 175 emit([:upload], %{size_bytes: size_bytes}, %{model_version_id: version.id})
  @@ -209,10 +240,10 @@ defmodule CrucibleModelRegistry do
209 240 end
210 241
211 242 defp get_or_create_model!(name, meta) do
212 - Repo.get_by(Model, name: name) ||
243 + repo().get_by(Model, name: name) ||
213 244 %Model{}
214 245 |> Model.changeset(Map.merge(%{name: name}, meta))
215 - |> Repo.insert!()
246 + |> repo().insert!()
216 247 end
217 248
218 249 defp insert_artifacts!(_version, []), do: []
  @@ -226,7 +257,7 @@ defmodule CrucibleModelRegistry do
226 257
227 258 %Artifact{}
228 259 |> Artifact.changeset(artifact_attrs)
229 - |> Repo.insert!()
260 + |> repo().insert!()
230 261 end)
231 262 end
  @@ -1,16 +1,35 @@
1 1 defmodule CrucibleModelRegistry.Application do
2 - @moduledoc "OTP application for crucible_model_registry."
2 + @moduledoc """
3 + OTP application for CrucibleModelRegistry.
4 +
5 + Note: The Repo is NOT started automatically. Host applications should:
6 + 1. Configure the repo: `config :crucible_model_registry, repo: MyApp.Repo`
7 + 2. Start their own Repo in their supervision tree
8 +
9 + For backwards compatibility, set `start_repo: true` to auto-start
10 + `CrucibleModelRegistry.Repo` (requires database config).
11 + """
3 12
4 13 use Application
5 14
6 15 @doc false
7 16 @impl true
8 17 def start(_type, _args) do
9 - children = [
10 - CrucibleModelRegistry.Repo
11 - ]
18 + children =
19 + []
20 + |> maybe_add_legacy_repo()
12 21
13 22 opts = [strategy: :one_for_one, name: CrucibleModelRegistry.Supervisor]
14 23 Supervisor.start_link(children, opts)
15 24 end
25 +
26 + # Legacy support: only start internal Repo if explicitly enabled
27 + # New pattern: host app provides repo via config :crucible_model_registry, repo: MyApp.Repo
28 + defp maybe_add_legacy_repo(children) do
29 + if Application.get_env(:crucible_model_registry, :start_repo, false) do
30 + [CrucibleModelRegistry.Repo | children]
31 + else
32 + children
33 + end
34 + end
16 35 end
Loading more files…