Current section
7 Versions
Jump to
Current section
7 Versions
Compare versions
13
files changed
+478
additions
-199
deletions
| @@ -3,8 +3,8 @@ | |
| 3 3 | {<<"description">>,<<"Genserver based content indexer">>}. |
| 4 4 | {<<"elixir">>,<<"~> 1.5">>}. |
| 5 5 | {<<"files">>, |
| 6 | - [<<"lib/content_indexer.ex">>,<<"lib/services/calculator.ex">>, |
| 7 | - <<"lib/services/index.ex">>,<<"lib/services/indexer.ex">>, |
| 6 | + [<<"lib/content_indexer.ex">>,<<"lib/index/index.ex">>, |
| 7 | + <<"lib/index/indexer.ex">>,<<"lib/services/calculator.ex">>, |
| 8 8 | <<"lib/services/list_checker_server.ex">>, |
| 9 9 | <<"lib/services/list_checker_worker.ex">>,<<"lib/services/pre_process.ex">>, |
| 10 10 | <<"lib/services/search_utils.ex">>,<<"lib/services/simularity.ex">>, |
| @@ -30,4 +30,4 @@ | |
| 30 30 | {<<"optional">>,false}, |
| 31 31 | {<<"repository">>,<<"hexpm">>}, |
| 32 32 | {<<"requirement">>,<<"~> 1.0">>}]]}. |
| 33 | - {<<"version">>,<<"0.1.0">>}. |
| 33 | + {<<"version">>,<<"0.2.0">>}. |
| @@ -13,7 +13,7 @@ defmodule ContentIndexer do | |
| 13 13 | worker(ContentIndexer.Services.Calculator, []), |
| 14 14 | worker(ContentIndexer.Services.ListCheckerServer, []), |
| 15 15 | worker(ContentIndexer.Services.ListCheckerWorker, []), |
| 16 | - worker(ContentIndexer.Services.Indexer, []) |
| 16 | + worker(ContentIndexer.Indexer, []) |
| 17 17 | ] |
| 18 18 | |
| 19 19 | # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html |
| @@ -0,0 +1,29 @@ | |
| 1 | + defmodule ContentIndexer.Index do |
| 2 | + @moduledoc """ |
| 3 | + struct to store the details of what data is held in the index |
| 4 | + It provides a `new/2` function for instantiating the struct that includes a generated UUID |
| 5 | + """ |
| 6 | + alias Ecto.UUID |
| 7 | + alias ContentIndexer.Index |
| 8 | + |
| 9 | + defstruct uuid: "", file_name: "", tokens: [], term_weights: %{} |
| 10 | + |
| 11 | + @doc """ |
| 12 | + Instantiates a new Index struct |
| 13 | + |
| 14 | + ## Parameters |
| 15 | + |
| 16 | + - file_name: String that represents the file that has the content to be indexed |
| 17 | + - tokens: List of Strings that are the tokenised content |
| 18 | + |
| 19 | + ## Example |
| 20 | + |
| 21 | + iex> ContentIndexer.Index.new("test_file.md", ["bread", "butter", "jam", "mustard"]) |
| 22 | + %ContentIndexer.Index{file_name: "test_file.md", term_weights: %{}, tokens: ["bread", "butter", "jam", "mustard"], uuid: "e080d012-f89b-434c-964f-ddad9b8c2e20"} |
| 23 | + """ |
| 24 | + def new(file_name, tokens) when is_list(tokens) do |
| 25 | + new_uuid = UUID.generate() |
| 26 | + %Index{file_name: file_name, tokens: tokens, uuid: new_uuid} |
| 27 | + end |
| 28 | + |
| 29 | + end |
| @@ -0,0 +1,181 @@ | |
| 1 | + defmodule ContentIndexer.Indexer do |
| 2 | + @moduledoc """ |
| 3 | + ** Summary ** |
| 4 | + Indexer is a Genserver that holds the index state - basically a list of index structs that have the filename, tokens and weights |
| 5 | + Each time an index struct is added to the server/index the weightings are re-calculated. Since they are stored in memory the index searching is fast |
| 6 | + """ |
| 7 | + |
| 8 | + use GenServer |
| 9 | + alias ContentIndexer.{Index, Services.Calculator} |
| 10 | + |
| 11 | + def start_link do |
| 12 | + GenServer.start_link(__MODULE__, :ok, [name: __MODULE__]) |
| 13 | + end |
| 14 | + |
| 15 | + def init(:ok) do |
| 16 | + {:ok, init_indexer()} |
| 17 | + end |
| 18 | + |
| 19 | + @doc """ |
| 20 | + Initialises the Index with an empty list |
| 21 | + """ |
| 22 | + def init_indexer do |
| 23 | + IO.puts "\nInitialising Indexer\n" |
| 24 | + [] # initialise with an empty list |
| 25 | + end |
| 26 | + |
| 27 | + # - client functions |
| 28 | + |
| 29 | + @doc """ |
| 30 | + Retrieves a list of all the tokens in the entire index |
| 31 | + |
| 32 | + ## Example |
| 33 | + |
| 34 | + iex> ContentIndexer.Indexer.corpus_of_tokens |
| 35 | + {:ok, |
| 36 | + [["orange", "fruit", "basket", "apples"], |
| 37 | + ["bread", "butter", "jam", "mustard"]]} |
| 38 | + """ |
| 39 | + def corpus_of_tokens do |
| 40 | + GenServer.call(__MODULE__, {:corpus_of_tokens}) |
| 41 | + end |
| 42 | + |
| 43 | + @doc """ |
| 44 | + Re calculates all the term_weights on the entire index |
| 45 | + |
| 46 | + ## Example |
| 47 | + |
| 48 | + iex>ContentIndexer.Indexer.calculate() |
| 49 | + {:ok, |
| 50 | + [%ContentIndexer.Index{file_name: "test_file_3.md", |
| 51 | + term_weights: [{"orange", 0.0}, {"fruit", 0.0}, {"basket", 0.0}, |
| 52 | + {"apples", 0.0}], tokens: ["orange", "fruit", "basket", "apples"], |
| 53 | + uuid: "2c600089-b35d-4667-a146-4635bd282811"}, |
| 54 | + %ContentIndexer.Index{file_name: "test_file_2.md", |
| 55 | + term_weights: [{"orange", 0.0}, {"fruit", 0.0}, {"basket", 0.0}, |
| 56 | + {"apples", 0.0}], tokens: ["orange", "fruit", "basket", "apples"], |
| 57 | + uuid: "c62c65be-4ac6-46bc-9597-2d70c65fa1a0"}, |
| 58 | + %ContentIndexer.Index{file_name: "test_file.md", |
| 59 | + term_weights: [{"bread", 0.1013662770270411}, {"butter", 0.1013662770270411}, |
| 60 | + {"jam", 0.1013662770270411}, {"mustard", 0.1013662770270411}], |
| 61 | + tokens: ["bread", "butter", "jam", "mustard"], |
| 62 | + uuid: "18693629-bfa9-4ffc-8fe8-ebc0c5c72c7b"}]} |
| 63 | + """ |
| 64 | + def calculate do |
| 65 | + GenServer.call(__MODULE__, {:calculate}) |
| 66 | + end |
| 67 | + |
| 68 | + @doc """ |
| 69 | + Adds a new file_name and associated list of tokens to the index |
| 70 | + |
| 71 | + ## Parameters |
| 72 | + |
| 73 | + - file_name: String that represents the file that has the content to be indexed |
| 74 | + - tokens: List of Strings that are the tokenised content |
| 75 | + |
| 76 | + ## Example |
| 77 | + |
| 78 | + iex> ContentIndexer.Indexer.add("test_file.md", ["bread", "butter", "jam", "mustard"]) |
| 79 | + {:ok, |
| 80 | + [%ContentIndexer.Index{file_name: "test_file.md", |
| 81 | + term_weights: [{"bread", -0.17328679513998632}, |
| 82 | + {"butter", -0.17328679513998632}, {"jam", -0.17328679513998632}, |
| 83 | + {"mustard", -0.17328679513998632}], |
| 84 | + tokens: ["bread", "butter", "jam", "mustard"], |
| 85 | + uuid: "18693629-bfa9-4ffc-8fe8-ebc0c5c72c7b"}]} |
| 86 | + """ |
| 87 | + def add(file_name, tokens) when is_list(tokens) do |
| 88 | + item = Index.new(file_name, tokens) |
| 89 | + GenServer.call(__MODULE__, {:add, item}) |
| 90 | + end |
| 91 | + |
| 92 | + @doc """ |
| 93 | + Returns a nested list of all the individual index items containing their file_name and associated tokens with weights |
| 94 | + |
| 95 | + ## Example |
| 96 | + |
| 97 | + iex> ContentIndexer.Indexer.documents() |
| 98 | + {:ok, |
| 99 | + [{"test_file_3.md", |
| 100 | + [{"orange", 0.0}, {"fruit", 0.0}, {"basket", 0.0}, {"apples", 0.0}]}, |
| 101 | + {"test_file_2.md", |
| 102 | + [{"orange", 0.0}, {"fruit", 0.0}, {"basket", 0.0}, {"apples", 0.0}]}, |
| 103 | + {"test_file.md", |
| 104 | + [{"bread", 0.1013662770270411}, {"butter", 0.1013662770270411}, |
| 105 | + {"jam", 0.1013662770270411}, {"mustard", 0.1013662770270411}]}]} |
| 106 | + """ |
| 107 | + def documents do |
| 108 | + GenServer.call(__MODULE__, {:documents}) |
| 109 | + end |
| 110 | + |
| 111 | + @doc """ |
| 112 | + Retrieves the entire index |
| 113 | + """ |
| 114 | + def retrieve_index do |
| 115 | + GenServer.call(__MODULE__, {:state}) |
| 116 | + end |
| 117 | + |
| 118 | + @doc """ |
| 119 | + Resets the index with an empty list |
| 120 | + """ |
| 121 | + def reset_index do |
| 122 | + GenServer.call(__MODULE__, {:reset_index}) |
| 123 | + end |
| 124 | + |
| 125 | + # - internal genserver call handler methods |
| 126 | + |
| 127 | + def handle_call({:reset_index}, _from, _state) do |
| 128 | + {:reply, {:ok, []}, []} |
| 129 | + end |
| 130 | + |
| 131 | + def handle_call({:state}, _from, state) do |
| 132 | + {:reply, {:ok, state}, state} |
| 133 | + end |
| 134 | + |
| 135 | + def handle_call({:add, index}, _from, state) do |
| 136 | + state = [index | state] |
| 137 | + # - after adding to the index we need to re-calculate the weightings |
| 138 | + weighted_state = calculate_weights(state) |
| 139 | + {:reply, {:ok, weighted_state}, weighted_state} |
| 140 | + end |
| 141 | + |
| 142 | + def handle_call({:calculate}, _from, state) do |
| 143 | + weighted_state = calculate_weights(state) |
| 144 | + {:reply, {:ok, weighted_state}, weighted_state} |
| 145 | + end |
| 146 | + |
| 147 | + def handle_call({:corpus_of_tokens}, _from, state) do |
| 148 | + corpus_of_tokens = get_corpus_of_tokens(state) |
| 149 | + {:reply, {:ok, corpus_of_tokens}, state} |
| 150 | + end |
| 151 | + |
| 152 | + def handle_call({:documents}, _from, state) do |
| 153 | + documents = retrieve_as_documents(state) |
| 154 | + {:reply, {:ok, documents}, state} |
| 155 | + end |
| 156 | + |
| 157 | + # private methods used by the internal handle_call methods |
| 158 | + |
| 159 | + defp retrieve_as_documents(state) do |
| 160 | + state |
| 161 | + |> Enum.map(fn(d) -> |
| 162 | + {d.file_name, d.term_weights} |
| 163 | + end) |
| 164 | + end |
| 165 | + |
| 166 | + defp calculate_weights(state) do |
| 167 | + corpus = get_corpus_of_tokens(state) |
| 168 | + state |
| 169 | + |> Enum.map(fn(i) -> |
| 170 | + {:ok, calculated_weights} = Calculator.calculate_content_indexer_documents(i.tokens, corpus) |
| 171 | + %Index{file_name: i.file_name, uuid: i.uuid, tokens: i.tokens, term_weights: calculated_weights} |
| 172 | + end) |
| 173 | + end |
| 174 | + |
| 175 | + defp get_corpus_of_tokens(state) do |
| 176 | + state |
| 177 | + |> Enum.map(fn(t) -> |
| 178 | + t.tokens |
| 179 | + end) |
| 180 | + end |
| 181 | + end |
| @@ -44,14 +44,6 @@ defmodule ContentIndexer.Services.Calculator do | |
| 44 44 | IO.puts "\nInitialising Calculator\n" |
| 45 45 | end |
| 46 46 | |
| 47 | - def handle_call({:state}, _from, state) do |
| 48 | - {:reply, {:ok, state}, state} |
| 49 | - end |
| 50 | - |
| 51 | - def handle_call({:total, _count}, _from, state) do |
| 52 | - {:reply, {:ok, state}, state} |
| 53 | - end |
| 54 | - |
| 55 47 | def total(count) do |
| 56 48 | GenServer.call(__MODULE__, {:total, count}) |
| 57 49 | end |
| @@ -59,17 +51,21 @@ defmodule ContentIndexer.Services.Calculator do | |
| 59 51 | @doc """ |
| 60 52 | calculates the content_indexer |
| 61 53 | |
| 62 | - iex> ContentIndexerValidateService.calculate_tokens_againts_corpus( |
| 63 | - "bread,butter,jam", |
| 64 | - ["red,brown,jam","blue,green,butter","pink,green,bread,jam"] |
| 65 | - ) |
| 66 | - {:ok, |
| 67 | - [ |
| 68 | - {"bread", 0.13515503603605478}, |
| 69 | - {"butter", 0.13515503603605478}, |
| 70 | - {"jam", 0.0} |
| 71 | - ] |
| 72 | - } |
| 54 | + ## Parameters |
| 55 | + |
| 56 | + - content: String of tokens to be indexed |
| 57 | + - corpus: List of String tokens representing the corpus |
| 58 | + |
| 59 | + ## Example |
| 60 | + |
| 61 | + iex> ContentIndexerValidateService.calculate_tokens_againts_corpus("bread,butter,jam", ["red,brown,jam","blue,green,butter","pink,green,bread,jam"]) |
| 62 | + {:ok, |
| 63 | + [ |
| 64 | + {"bread", 0.13515503603605478}, |
| 65 | + {"butter", 0.13515503603605478}, |
| 66 | + {"jam", 0.0} |
| 67 | + ] |
| 68 | + } |
| 73 69 | """ |
| 74 70 | def calculate_tokens_againts_corpus(content, corpus) do |
| 75 71 | token_list = Tfidf.calculate_all(content, corpus, &String.split(&1, ",")) |
| @@ -80,8 +76,14 @@ defmodule ContentIndexer.Services.Calculator do | |
| 80 76 | calculates the word count for each token in the list of tokens representing the document |
| 81 77 | and returns a list of the tokens with their respective word counts |
| 82 78 | |
| 83 | - iex> ContentIndexerService.calculate_token_count_document(["bread","butter","jam","jam","bread","bread"]) |
| 84 | - {:ok, [bread: 3, butter: 1, jam: 2]} |
| 79 | + ## Parameters |
| 80 | + |
| 81 | + - tokens: List of tokens to be indexed |
| 82 | + |
| 83 | + ## Example |
| 84 | + |
| 85 | + iex> ContentIndexerService.calculate_token_count_document(["bread","butter","jam","jam","bread","bread"]) |
| 86 | + {:ok, [bread: 3, butter: 1, jam: 2]} |
| 85 87 | """ |
| 86 88 | def calculate_token_count_document(tokens) do |
| 87 89 | token_stream = Stream.map(tokens, fn(token) -> |
| @@ -95,8 +97,14 @@ defmodule ContentIndexer.Services.Calculator do | |
| 95 97 | calculates the term frequency for each token in the list of tokens representing the document |
| 96 98 | and returns a list of the tokens with their respective term frequencies |
| 97 99 | |
| 98 | - iex> ContentIndexerService.calculate_tf_document(["bread","butter","jam","jam","bread","bread"]) |
| 99 | - {:ok, [bread: 0.5, butter: 0.16666666666666666, jam: 0.3333333333333333]} |
| 100 | + ## Parameters |
| 101 | + |
| 102 | + - tokens: List of tokens to be indexed |
| 103 | + |
| 104 | + ## Example |
| 105 | + |
| 106 | + iex> ContentIndexerService.calculate_tf_document(["bread","butter","jam","jam","bread","bread"]) |
| 107 | + {:ok, [bread: 0.5, butter: 0.16666666666666666, jam: 0.3333333333333333]} |
| 100 108 | """ |
| 101 109 | def calculate_tf_document(tokens) do |
| 102 110 | token_stream = Stream.map(tokens, fn(token) -> |
| @@ -109,10 +117,14 @@ defmodule ContentIndexer.Services.Calculator do | |
| 109 117 | @doc """ |
| 110 118 | calculates the content_indexer weights for each token in the query - weights the query against itself |
| 111 119 | |
| 112 | - iex> ContentIndexerService.calculate_content_indexer_query( |
| 113 | - ["bread","butter","jam"] |
| 114 | - ) |
| 115 | - {:ok, [bread: 0.0, butter: 0.0, jam: 0.0]} |
| 120 | + ## Parameters |
| 121 | + |
| 122 | + - tokens: List of tokens to be indexed |
| 123 | + |
| 124 | + ## Example |
| 125 | + |
| 126 | + iex> ContentIndexerService.calculate_content_indexer_query(["bread","butter","jam"]) |
| 127 | + {:ok, [bread: 0.0, butter: 0.0, jam: 0.0]} |
| 116 128 | """ |
| 117 129 | def calculate_content_indexer_query(tokens) do |
| 118 130 | tokenized_tokens = case tokens do |
| @@ -132,11 +144,21 @@ defmodule ContentIndexer.Services.Calculator do | |
| 132 144 | @doc """ |
| 133 145 | calculates the content_indexer weights for each token in the list of tokens against the corpus of tokens |
| 134 146 | |
| 135 | - iex> ContentIndexerService.calculate_content_indexer_documents( |
| 136 | - ["bread","butter","jam"], |
| 137 | - [["red","brown","jam"],["blue","green","butter"],["pink","green","bread","jam"]] |
| 138 | - ) |
| 139 | - {:ok, [bread: 0.3662040962227032, butter: 0.3662040962227032,jam: 0.3662040962227032]} |
| 147 | + ## Parameters |
| 148 | + |
| 149 | + - tokens: List of tokens to be indexed |
| 150 | + - corpus_of_tokens: List of lists representing the corpus of all tokens |
| 151 | + |
| 152 | + ## Example |
| 153 | + |
| 154 | + iex> ContentIndexerService.calculate_content_indexer_documents( |
| 155 | + ["bread","butter","jam"], |
| 156 | + [ |
| 157 | + ["red","brown","jam"], |
| 158 | + ["blue","green","butter"], |
| 159 | + ["pink","green","bread","jam"] |
| 160 | + ]) |
| 161 | + {:ok, [bread: 0.3662040962227032, butter: 0.3662040962227032,jam: 0.3662040962227032]} |
| 140 162 | """ |
| 141 163 | def calculate_content_indexer_documents(tokens, corpus_of_tokens) do |
| 142 164 | corpus_size = length(corpus_of_tokens) # this is so we can avoid calculating it again! |
| @@ -148,6 +170,26 @@ defmodule ContentIndexer.Services.Calculator do | |
| 148 170 | end |
| 149 171 | end |
| 150 172 | |
| 173 | + @doc """ |
| 174 | + calculates the content_indexer weights for each token in the list of tokens against the corpus of tokens |
| 175 | + |
| 176 | + ## Parameters |
| 177 | + |
| 178 | + - tokens: List of tokens to be indexed |
| 179 | + - corpus_of_tokens: List of lists representing the corpus of all tokens |
| 180 | + - corpus_size: Integer with the size of the corpus_of_tokens - just so we can avoid re-calculating it |
| 181 | + |
| 182 | + ## Example |
| 183 | + |
| 184 | + iex> ContentIndexerService.calculate_content_indexer_documents( |
| 185 | + ["bread","butter","jam"], |
| 186 | + [ |
| 187 | + ["red","brown","jam"], |
| 188 | + ["blue","green","butter"], |
| 189 | + ["pink","green","bread","jam"] |
| 190 | + ]) |
| 191 | + {:ok, [bread: 0.3662040962227032, butter: 0.3662040962227032,jam: 0.3662040962227032]} |
| 192 | + """ |
| 151 193 | def calculate_content_indexer_documents(tokens, corpus_of_tokens, corpus_size) do |
| 152 194 | case corpus_size do |
| 153 195 | 1 -> |
| @@ -157,6 +199,39 @@ defmodule ContentIndexer.Services.Calculator do | |
| 157 199 | end |
| 158 200 | end |
| 159 201 | |
| 202 | + @doc """ |
| 203 | + simple function to check if an item is contained in the list |
| 204 | + |
| 205 | + ## Parameters |
| 206 | + |
| 207 | + - list: List of any type |
| 208 | + - item: Any type of item stored in the list |
| 209 | + |
| 210 | + ## Example |
| 211 | + |
| 212 | + iex> ContentIndexerService.calculate_content_indexer_documents( |
| 213 | + ["bread","butter","jam"], |
| 214 | + [ |
| 215 | + ["red","brown","jam"], |
| 216 | + ["blue","green","butter"], |
| 217 | + ["pink","green","bread","jam"] |
| 218 | + ]) |
| 219 | + {:ok, [bread: 0.3662040962227032, butter: 0.3662040962227032,jam: 0.3662040962227032]} |
| 220 | + """ |
| 221 | + def list_contains(list, item), do: Enum.find(list, &(item == &1)) != nil |
| 222 | + |
| 223 | + # - internal genserver call handler methods |
| 224 | + |
| 225 | + def handle_call({:state}, _from, state) do |
| 226 | + {:reply, {:ok, state}, state} |
| 227 | + end |
| 228 | + |
| 229 | + def handle_call({:total, _count}, _from, state) do |
| 230 | + {:reply, {:ok, state}, state} |
| 231 | + end |
| 232 | + |
| 233 | + # - private methods |
| 234 | + |
| 160 235 | defp calculate_content_indexer_documents_single(tokens, corpus_of_tokens) do |
| 161 236 | token_content_indexer_counts = tokens |
| 162 237 | |> Enum.uniq |
| @@ -197,10 +272,6 @@ defmodule ContentIndexer.Services.Calculator do | |
| 197 272 | total |
| 198 273 | end |
| 199 274 | |
| 200 | - def list_contains(list, item) do |
| 201 | - Enum.find(list, fn(cur_item) -> item == cur_item end) != nil |
| 202 | - end |
| 203 | - |
| 204 275 | defp idf(word, corpus_of_tokens) do |
| 205 276 | :math.log(length(corpus_of_tokens) / (1 + n_containing(word, corpus_of_tokens))) |
| 206 277 | end |
Loading more files…