Packages

portfolio_index

0.3.0
0.4.0 0.3.1 0.3.0 0.2.0 0.1.1 0.1.0

Production adapters and pipelines for PortfolioCore. Vector stores, graph stores, embedders, Broadway pipelines, and advanced RAG strategies.

HexDocs

Current section

6 Versions

Jump to
Readme
6 Versions
17 Dependencies
3 Dependants
Files
Activity
Hide whitespace

Compare versions

23 files changed
+4304 additions
-61 deletions
  @@ -7,6 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7 7
8 8 ## [Unreleased]
9 9
10 + ## [0.3.0] - 2025-12-28
11 +
12 + ### Added
13 +
14 + #### Chunker Adapters
15 + - Character chunker with boundary modes (`:word`, `:sentence`, `:none`)
16 + - Paragraph chunker with intelligent merging and splitting
17 + - Sentence chunker with NLP tokenization and abbreviation handling
18 + - Semantic chunker using embedding-based similarity grouping
19 +
20 + #### GraphRAG Components
21 + - `CommunityDetector` with label propagation algorithm
22 + - `CommunitySummarizer` with LLM-based summarization and embedding generation
23 + - `EntityExtractor` with batch support and entity resolution
24 +
25 + #### Graph Store Enhancements (Neo4j)
26 + - `EntitySearch` module for vector-based entity search
27 + - `Community` module for community CRUD and vector search
28 + - `Traversal` module for BFS, subgraph extraction, and path finding
29 +
30 + #### Reranker Adapters
31 + - LLM-based reranker with customizable prompts
32 + - Passthrough reranker for testing and baselines
33 +
34 + #### Retriever Enhancements
35 + - GraphRAG local/global/hybrid search modes
36 + - PostgreSQL tsvector-based full-text search
37 +
38 + ### Changed
39 +
40 + - GraphRAG strategy now supports `:mode` option (`:local`, `:global`, `:hybrid`)
41 + - Updated portfolio_core dependency to path reference for development
42 +
10 43 ## [0.2.0] - 2025-12-27
11 44
12 45 ### Added
  @@ -48,7 +81,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
48 81
49 82 - Initial release of PortfolioIndex
50 83
51 - [Unreleased]: https://github.com/nshkrdotcom/portfolio_index/compare/v0.2.0...HEAD
84 + [Unreleased]: https://github.com/nshkrdotcom/portfolio_index/compare/v0.3.0...HEAD
85 + [0.3.0]: https://github.com/nshkrdotcom/portfolio_index/compare/v0.2.0...v0.3.0
52 86 [0.2.0]: https://github.com/nshkrdotcom/portfolio_index/compare/v0.1.1...v0.2.0
53 87 [0.1.1]: https://github.com/nshkrdotcom/portfolio_index/compare/v0.1.0...v0.1.1
54 88 [0.1.0]: https://github.com/nshkrdotcom/portfolio_index/releases/tag/v0.1.0
  @@ -19,8 +19,8 @@
19 19
20 20 Portfolio Index implements the port specifications defined in [Portfolio Core](https://github.com/nshkrdotcom/portfolio_core), providing:
21 21
22 - - **Vector Store Adapters** - pgvector (PostgreSQL)
23 - - **Graph Store Adapters** - Neo4j via boltx
22 + - **Vector Store Adapters** - pgvector (PostgreSQL + fulltext hybrid)
23 + - **Graph Store Adapters** - Neo4j via boltx + community operations
24 24 - **Embedding Providers** - Google Gemini
25 25 - **LLM Providers** - Google Gemini, Anthropic Claude, OpenAI GPT/o1
26 26 - **Broadway Pipelines** - Ingestion and embedding with backpressure
  @@ -1,7 +1,7 @@
1 1 {<<"links">>,
2 2 [{<<"GitHub">>,<<"https://github.com/nshkrdotcom/portfolio_index">>}]}.
3 3 {<<"name">>,<<"portfolio_index">>}.
4 - {<<"version">>,<<"0.2.0">>}.
4 + {<<"version">>,<<"0.3.0">>}.
5 5 {<<"description">>,
6 6 <<"Production adapters and pipelines for PortfolioCore. Vector stores, graph stores, embedders, Broadway pipelines, and advanced RAG strategies.">>}.
7 7 {<<"elixir">>,<<"~> 1.15">>}.
  @@ -15,10 +15,20 @@
15 15 <<"lib/portfolio_index/adapters/graph_store/neo4j.ex">>,
16 16 <<"lib/portfolio_index/adapters/graph_store/neo4j">>,
17 17 <<"lib/portfolio_index/adapters/graph_store/neo4j/schema.ex">>,
18 + <<"lib/portfolio_index/adapters/graph_store/neo4j/entity_search.ex">>,
19 + <<"lib/portfolio_index/adapters/graph_store/neo4j/traversal.ex">>,
20 + <<"lib/portfolio_index/adapters/graph_store/neo4j/community.ex">>,
18 21 <<"lib/portfolio_index/adapters/document_store">>,
19 22 <<"lib/portfolio_index/adapters/document_store/postgres.ex">>,
23 + <<"lib/portfolio_index/adapters/reranker">>,
24 + <<"lib/portfolio_index/adapters/reranker/passthrough.ex">>,
25 + <<"lib/portfolio_index/adapters/reranker/llm.ex">>,
20 26 <<"lib/portfolio_index/adapters/chunker">>,
27 + <<"lib/portfolio_index/adapters/chunker/sentence.ex">>,
28 + <<"lib/portfolio_index/adapters/chunker/semantic.ex">>,
29 + <<"lib/portfolio_index/adapters/chunker/character.ex">>,
21 30 <<"lib/portfolio_index/adapters/chunker/recursive.ex">>,
31 + <<"lib/portfolio_index/adapters/chunker/paragraph.ex">>,
22 32 <<"lib/portfolio_index/adapters/embedder">>,
23 33 <<"lib/portfolio_index/adapters/embedder/gemini.ex">>,
24 34 <<"lib/portfolio_index/adapters/embedder/open_ai.ex">>,
  @@ -27,8 +37,15 @@
27 37 <<"lib/portfolio_index/adapters/llm/gemini.ex">>,
28 38 <<"lib/portfolio_index/adapters/llm/open_ai.ex">>,
29 39 <<"lib/portfolio_index/adapters/vector_store">>,
40 + <<"lib/portfolio_index/adapters/vector_store/pgvector">>,
41 + <<"lib/portfolio_index/adapters/vector_store/pgvector/hybrid.ex">>,
42 + <<"lib/portfolio_index/adapters/vector_store/pgvector/fulltext.ex">>,
30 43 <<"lib/portfolio_index/adapters/vector_store/pgvector.ex">>,
31 44 <<"lib/portfolio_index/postgrex_types.ex">>,
45 + <<"lib/portfolio_index/graph_rag">>,
46 + <<"lib/portfolio_index/graph_rag/entity_extractor.ex">>,
47 + <<"lib/portfolio_index/graph_rag/community_detector.ex">>,
48 + <<"lib/portfolio_index/graph_rag/community_summarizer.ex">>,
32 49 <<"lib/portfolio_index/pipelines">>,
33 50 <<"lib/portfolio_index/pipelines/ingestion.ex">>,
34 51 <<"lib/portfolio_index/pipelines/embedding.ex">>,
  @@ -49,7 +66,7 @@
49 66 [[{<<"name">>,<<"portfolio_core">>},
50 67 {<<"app">>,<<"portfolio_core">>},
51 68 {<<"optional">>,false},
52 - {<<"requirement">>,<<"~> 0.2.0">>},
69 + {<<"requirement">>,<<"~> 0.3.0">>},
53 70 {<<"repository">>,<<"hexpm">>}],
54 71 [{<<"name">>,<<"ecto_sql">>},
55 72 {<<"app">>,<<"ecto_sql">>},
  @@ -10,7 +10,9 @@ defmodule PortfolioIndex do
10 10 ## Adapters
11 11
12 12 - Vector Stores: `PortfolioIndex.Adapters.VectorStore.Pgvector`
13 + (hybrid fulltext via `PortfolioIndex.Adapters.VectorStore.Pgvector.Hybrid`)
13 14 - Graph Stores: `PortfolioIndex.Adapters.GraphStore.Neo4j`
15 + (community ops via `PortfolioIndex.Adapters.GraphStore.Neo4j.Community`)
14 16 - Embedders: `PortfolioIndex.Adapters.Embedder.Gemini`
15 17 - LLMs: `PortfolioIndex.Adapters.LLM.Gemini`, `PortfolioIndex.Adapters.LLM.Anthropic`,
16 18 `PortfolioIndex.Adapters.LLM.OpenAI`
  @@ -0,0 +1,288 @@
1 + defmodule PortfolioIndex.Adapters.Chunker.Character do
2 + @moduledoc """
3 + Character-based text chunker with smart boundary handling.
4 +
5 + Implements the `PortfolioCore.Ports.Chunker` behaviour.
6 +
7 + ## Strategy
8 +
9 + Splits text at character boundaries with optional word/sentence preservation:
10 + - `:word` - Never split in the middle of a word (default)
11 + - `:sentence` - Try to split at sentence boundaries
12 + - `:none` - Split exactly at character count
13 +
14 + ## Example
15 +
16 + config = %{chunk_size: 500, chunk_overlap: 100, boundary: :word}
17 + {:ok, chunks} = Character.chunk(text, :plain, config)
18 + """
19 +
20 + @behaviour PortfolioCore.Ports.Chunker
21 +
22 + @default_chunk_size 1000
23 + @default_chunk_overlap 200
24 +
25 + @type boundary :: :word | :sentence | :none
26 +
27 + @impl true
28 + @spec chunk(String.t(), atom(), map()) :: {:ok, [map()]} | {:error, term()}
29 + def chunk(text, _format, config) do
30 + chunk_size = config[:chunk_size] || @default_chunk_size
31 + chunk_overlap = config[:chunk_overlap] || @default_chunk_overlap
32 + boundary = config[:boundary] || :word
33 +
34 + if String.trim(text) == "" do
35 + {:ok, []}
36 + else
37 + chunks = split_with_boundary(text, chunk_size, chunk_overlap, boundary)
38 +
39 + result =
40 + chunks
41 + |> Enum.with_index()
42 + |> Enum.map(fn {content, index} ->
43 + {start_byte, end_byte} = calculate_byte_positions(text, content, index, chunks)
44 +
45 + %{
46 + content: content,
47 + index: index,
48 + start_byte: start_byte,
49 + end_byte: end_byte,
50 + start_offset: start_byte,
51 + end_offset: end_byte,
52 + metadata: %{
53 + strategy: :character,
54 + boundary: boundary,
55 + char_count: String.length(content)
56 + }
57 + }
58 + end)
59 +
60 + {:ok, result}
61 + end
62 + end
63 +
64 + @impl true
65 + @spec estimate_chunks(String.t(), map()) :: non_neg_integer()
66 + def estimate_chunks(text, config) do
67 + chunk_size = config[:chunk_size] || @default_chunk_size
68 + chunk_overlap = config[:chunk_overlap] || @default_chunk_overlap
69 +
70 + text_length = String.length(text)
71 +
72 + if text_length <= chunk_size do
73 + 1
74 + else
75 + effective_chunk_size = max(chunk_size - chunk_overlap, 1)
76 + div(text_length, effective_chunk_size) + 1
77 + end
78 + end
79 +
80 + # Private functions
81 +
82 + @spec split_with_boundary(String.t(), pos_integer(), non_neg_integer(), boundary()) :: [
83 + String.t()
84 + ]
85 + defp split_with_boundary(text, chunk_size, overlap, :none) do
86 + # Direct character splitting without boundary awareness
87 + graphemes = String.graphemes(text)
88 + effective_step = max(chunk_size - overlap, 1)
89 +
90 + graphemes
91 + |> chunk_at_positions(chunk_size, effective_step)
92 + |> Enum.reject(&(String.trim(&1) == ""))
93 + end
94 +
95 + defp split_with_boundary(text, chunk_size, overlap, :word) do
96 + # Split at word boundaries
97 + split_at_word_boundaries(text, chunk_size, overlap)
98 + end
99 +
100 + defp split_with_boundary(text, chunk_size, overlap, :sentence) do
101 + # Split at sentence boundaries
102 + split_at_sentence_boundaries(text, chunk_size, overlap)
103 + end
104 +
105 + @spec chunk_at_positions([String.t()], pos_integer(), pos_integer()) :: [String.t()]
106 + defp chunk_at_positions(graphemes, chunk_size, step) do
107 + total = length(graphemes)
108 +
109 + if total <= chunk_size do
110 + [Enum.join(graphemes)]
111 + else
112 + 0
113 + |> Stream.iterate(&(&1 + step))
114 + |> Stream.take_while(&(&1 < total))
115 + |> Enum.map(fn start ->
116 + graphemes
117 + |> Enum.slice(start, chunk_size)
118 + |> Enum.join()
119 + end)
120 + end
121 + end
122 +
123 + @spec split_at_word_boundaries(String.t(), pos_integer(), non_neg_integer()) :: [String.t()]
124 + defp split_at_word_boundaries(text, chunk_size, overlap) do
125 + words = String.split(text, ~r/(\s+)/, include_captures: true)
126 +
127 + build_chunks_from_words(words, chunk_size, overlap)
128 + |> Enum.reject(&(String.trim(&1) == ""))
129 + end
130 +
131 + @spec build_chunks_from_words([String.t()], pos_integer(), non_neg_integer()) :: [String.t()]
132 + defp build_chunks_from_words(words, chunk_size, overlap) do
133 + {chunks, current, _} =
134 + Enum.reduce(words, {[], "", 0}, fn word, {chunks, current, current_len} ->
135 + word_len = String.length(word)
136 +
137 + cond do
138 + # Current chunk is empty, start with this word
139 + current == "" ->
140 + {chunks, word, word_len}
141 +
142 + # Adding this word would exceed chunk size
143 + current_len + word_len > chunk_size ->
144 + # Finalize current chunk and start new one with overlap
145 + overlap_text = get_word_overlap(current, overlap)
146 + {[current | chunks], overlap_text <> word, String.length(overlap_text) + word_len}
147 +
148 + # Add word to current chunk
149 + true ->
150 + {chunks, current <> word, current_len + word_len}
151 + end
152 + end)
153 +
154 + # Don't forget the last chunk
155 + if current != "" do
156 + Enum.reverse([current | chunks])
157 + else
158 + Enum.reverse(chunks)
159 + end
160 + end
161 +
162 + @spec get_word_overlap(String.t(), non_neg_integer()) :: String.t()
163 + defp get_word_overlap(_text, overlap) when overlap <= 0, do: ""
164 +
165 + defp get_word_overlap(text, overlap) do
166 + # Get the last `overlap` characters, but try to start at a word boundary
167 + text_len = String.length(text)
168 +
169 + if text_len <= overlap do
170 + text
171 + else
172 + start_pos = text_len - overlap
173 + overlap_text = String.slice(text, start_pos, overlap)
174 +
175 + # Try to find a word boundary (space) to start from
176 + case :binary.match(overlap_text, " ") do
177 + {pos, _} when pos < overlap ->
178 + String.slice(overlap_text, pos + 1, overlap)
179 +
180 + _ ->
181 + overlap_text
182 + end
183 + end
184 + end
185 +
186 + @spec split_at_sentence_boundaries(String.t(), pos_integer(), non_neg_integer()) :: [String.t()]
187 + defp split_at_sentence_boundaries(text, chunk_size, overlap) do
188 + # Split on sentence endings (. ! ?)
189 + sentences = split_into_sentences(text)
190 +
191 + build_chunks_from_sentences(sentences, chunk_size, overlap)
192 + |> Enum.reject(&(String.trim(&1) == ""))
193 + end
194 +
195 + @spec split_into_sentences(String.t()) :: [String.t()]
196 + defp split_into_sentences(text) do
197 + # Split on sentence boundaries while preserving the punctuation
198 + ~r/(?<=[.!?])\s+/
199 + |> Regex.split(text, include_captures: false)
200 + |> Enum.reject(&(String.trim(&1) == ""))
201 + end
202 +
203 + @spec build_chunks_from_sentences([String.t()], pos_integer(), non_neg_integer()) :: [
204 + String.t()
205 + ]
206 + defp build_chunks_from_sentences(sentences, chunk_size, overlap) do
207 + {chunks, current, _} =
208 + Enum.reduce(sentences, {[], "", 0}, fn sentence, {chunks, current, current_len} ->
209 + sentence_with_space = if current == "", do: sentence, else: " " <> sentence
210 + sentence_len = String.length(sentence_with_space)
211 +
212 + cond do
213 + # Single sentence exceeds chunk size, keep it as is
214 + current == "" and sentence_len > chunk_size ->
215 + {[sentence | chunks], "", 0}
216 +
217 + # Current chunk is empty, start with this sentence
218 + current == "" ->
219 + {chunks, sentence, sentence_len}
220 +
221 + # Adding this sentence would exceed chunk size
222 + current_len + sentence_len > chunk_size ->
223 + # Finalize current chunk and start new one with overlap
224 + overlap_text = get_sentence_overlap(current, overlap)
225 + new_current = String.trim(overlap_text <> " " <> sentence)
226 + {[current | chunks], new_current, String.length(new_current)}
227 +
228 + # Add sentence to current chunk
229 + true ->
230 + new_current = current <> sentence_with_space
231 + {chunks, new_current, String.length(new_current)}
232 + end
233 + end)
234 +
235 + # Don't forget the last chunk
236 + if current != "" do
237 + Enum.reverse([current | chunks])
238 + else
239 + Enum.reverse(chunks)
240 + end
241 + end
242 +
243 + @spec get_sentence_overlap(String.t(), non_neg_integer()) :: String.t()
244 + defp get_sentence_overlap(_text, overlap) when overlap <= 0, do: ""
245 +
246 + defp get_sentence_overlap(text, overlap) do
247 + # Get the last sentence(s) that fit within the overlap size
248 + sentences =
249 + text
250 + |> split_into_sentences()
251 + |> Enum.reverse()
252 +
253 + take_sentences_for_overlap(sentences, overlap, [])
254 + |> Enum.join(" ")
255 + end
256 +
257 + @spec take_sentences_for_overlap([String.t()], non_neg_integer(), [String.t()]) :: [String.t()]
258 + defp take_sentences_for_overlap([], _remaining, acc), do: acc
259 +
260 + defp take_sentences_for_overlap([sentence | rest], remaining, acc) do
261 + sentence_len = String.length(sentence)
262 +
263 + if sentence_len <= remaining do
264 + take_sentences_for_overlap(rest, remaining - sentence_len - 1, [sentence | acc])
265 + else
266 + acc
267 + end
268 + end
269 +
270 + @spec calculate_byte_positions(String.t(), String.t(), non_neg_integer(), [String.t()]) ::
271 + {non_neg_integer(), non_neg_integer()}
272 + defp calculate_byte_positions(text, content, index, _chunks) do
273 + # Find the actual byte position of this content in the original text
274 + start_byte = find_content_position(text, content, index)
275 + end_byte = start_byte + byte_size(content)
276 + {start_byte, end_byte}
277 + end
278 +
279 + @spec find_content_position(String.t(), String.t(), non_neg_integer()) :: non_neg_integer()
280 + defp find_content_position(text, content, _index) do
281 + # Find the byte offset of the content in the original text
282 + # For simplicity, use the first occurrence
283 + case :binary.match(text, content) do
284 + {pos, _} -> pos
285 + :nomatch -> 0
286 + end
287 + end
288 + end
Loading more files…