Packages
neural_net
1.0.0
NeuralNet is an A.I. library that allows for the construction and training of complex recurrent neural networks. Architectures such as LSTM or GRU can be specified in under 20 lines of code. Any neural network that can be built with the NeuralNet DSL can be trainined with automatically implemente...
Current section
Files
Jump to
Current section
Files
lib/sample_projects/language/parse.ex
defmodule SampleProjects.Language.Parse do
@moduledoc false
def parse(path) do
{:ok, text} = File.read path
# matches = Regex.scan(~r/(?:^|\s{2,})[A-Z][a-z]+\s[A-Za-z\s\-\$;:,]+\./, text)
text = Regex.replace(~r/\s+/, text, " ")
# IO.puts text
matches = Regex.scan(~r/(?<=^|\. )[A-Z][a-z]+(?: [A-Za-z;:,"]+)+(?=\.|!)/, text)
sentences = Enum.map(matches, fn [m] ->
s = Regex.replace(~r/(?!\a)\s+/, m, " ")
s = Regex.replace(~r/(?:^ |[;:,\."])/, s, "")
Regex.split(~r/ /, String.downcase(s))
end)
{
sentences,
Enum.reduce(sentences, MapSet.new, fn sentence, acc ->
Enum.reduce(sentence, acc, fn word, acc ->
MapSet.put(acc, word)
end)
end)
}
end
end