Current section
12 Versions
Jump to
Current section
12 Versions
Compare versions
9
files changed
+1357
additions
-67
deletions
| @@ -15,7 +15,7 @@ Add `paasaa` to your list of dependencies in `mix.exs`: | |
| 15 15 | |
| 16 16 | ```elixir |
| 17 17 | def deps do |
| 18 | - [{:paasaa, "~> 0.2.0"}] |
| 18 | + [{:paasaa, "~> 0.3.0"}] |
| 19 19 | end |
| 20 20 | ``` |
| 21 21 | |
| @@ -43,7 +43,7 @@ iex> Paasaa.all("Detect this!") | |
| 43 43 | ] |
| 44 44 | ``` |
| 45 45 | |
| 46 | - ## Derivation |
| 46 | + ## Derivation |
| 47 47 | |
| 48 48 | **Paasaa** is a derivative work from [Franc][] (JavaScript, MIT) by [Titus Wormer]. |
| @@ -1,13 +1,18 @@ | |
| 1 1 | {<<"app">>,<<"paasaa">>}. |
| 2 2 | {<<"build_tools">>,[<<"mix">>]}. |
| 3 3 | {<<"description">>,<<"Natural language detection">>}. |
| 4 | - {<<"elixir">>,<<"~> 1.5">>}. |
| 4 | + {<<"elixir">>,<<"~> 1.7">>}. |
| 5 5 | {<<"files">>, |
| 6 | - [<<"lib/paasaa.ex">>,<<"priv/languages.binary">>,<<"priv/scripts.binary">>, |
| 7 | - <<"mix.exs">>,<<"README.md">>,<<"LICENSE">>]}. |
| 6 | + [<<"lib/paasaa.ex">>,<<"priv">>,<<"priv/languages.json">>, |
| 7 | + <<"priv/scripts.json">>,<<"priv/support.json">>,<<"mix.exs">>, |
| 8 | + <<"README.md">>,<<"LICENSE">>]}. |
| 8 9 | {<<"licenses">>,[<<"MIT">>]}. |
| 9 10 | {<<"links">>,[{<<"GitHub">>,<<"https://github.com/minibikini/paasaa">>}]}. |
| 10 | - {<<"maintainers">>,[<<"Egor Kislitsyn">>]}. |
| 11 11 | {<<"name">>,<<"paasaa">>}. |
| 12 | - {<<"requirements">>,[]}. |
| 13 | - {<<"version">>,<<"0.2.2">>}. |
| 12 | + {<<"requirements">>, |
| 13 | + [[{<<"app">>,<<"exjsx">>}, |
| 14 | + {<<"name">>,<<"exjsx">>}, |
| 15 | + {<<"optional">>,false}, |
| 16 | + {<<"repository">>,<<"hexpm">>}, |
| 17 | + {<<"requirement">>,<<"~> 4.0">>}]]}. |
| 18 | + {<<"version">>,<<"0.3.0">>}. |
| @@ -9,30 +9,26 @@ defmodule Paasaa do | |
| 9 9 | |
| 10 10 | """ |
| 11 11 | |
| 12 | - @scripts "./priv/scripts.binary" |
| 13 | - |> File.read! |
| 14 | - |> :erlang.binary_to_term |
| 12 | + @scripts Paasaa.Data.scripts() |
| 15 13 | |
| 16 | - @languages "./priv/languages.binary" |
| 17 | - |> File.read! |
| 18 | - |> :erlang.binary_to_term |
| 14 | + @languages Paasaa.Data.languages() |
| 19 15 | |
| 20 16 | @max_difference 300 |
| 21 17 | |
| 22 | - @type result :: [{language :: String.t, score :: number}] |
| 18 | + @type result :: [{language :: String.t(), score :: number}] |
| 23 19 | |
| 24 20 | @type options :: [ |
| 25 | - min_length: integer, |
| 26 | - max_length: integer, |
| 27 | - whitelist: [String.t], |
| 28 | - blacklist: [String.t] |
| 29 | - ] |
| 21 | + min_length: integer, |
| 22 | + max_length: integer, |
| 23 | + whitelist: [String.t()], |
| 24 | + blacklist: [String.t()] |
| 25 | + ] |
| 30 26 | |
| 31 27 | @default_options [ |
| 32 28 | min_length: 10, |
| 33 29 | max_length: 2048, |
| 34 30 | whitelist: [], |
| 35 | - blacklist: [], |
| 31 | + blacklist: [] |
| 36 32 | ] |
| 37 33 | |
| 38 34 | @doc """ |
| @@ -69,11 +65,11 @@ defmodule Paasaa do | |
| 69 65 | "und" |
| 70 66 | """ |
| 71 67 | |
| 72 | - @spec detect(str :: String.t, options) :: language :: String.t |
| 68 | + @spec detect(str :: String.t(), options) :: language :: String.t() |
| 73 69 | def detect(str, options \\ @default_options) do |
| 74 70 | str |
| 75 71 | |> all(options) |
| 76 | - |> List.first |
| 72 | + |> List.first() |
| 77 73 | |> elem(0) |
| 78 74 | end |
| 79 75 | |
| @@ -99,12 +95,13 @@ defmodule Paasaa do | |
| 99 95 | ] |
| 100 96 | """ |
| 101 97 | |
| 102 | - @spec all(str :: String.t, options) :: result |
| 98 | + @spec all(str :: String.t(), options) :: result |
| 103 99 | def all(str, options \\ @default_options) |
| 104 100 | def all("", _), do: und() |
| 105 101 | def all(nil, _), do: und() |
| 102 | + |
| 106 103 | def all(str, options) do |
| 107 | - options = Keyword.merge @default_options, options |
| 104 | + options = Keyword.merge(@default_options, options) |
| 108 105 | |
| 109 106 | if String.length(str) < options[:min_length] do |
| 110 107 | und() |
| @@ -113,27 +110,30 @@ defmodule Paasaa do | |
| 113 110 | end |
| 114 111 | end |
| 115 112 | |
| 116 | - @spec process(str :: String.t, options) :: result |
| 113 | + @spec process(str :: String.t(), options) :: result |
| 117 114 | defp process(str, options) do |
| 118 | - str = String.slice str, 0, options[:max_length] |
| 115 | + str = String.slice(str, 0, options[:max_length]) |
| 119 116 | |
| 120 | - {script, count} = detect_script str |
| 117 | + {script, count} = detect_script(str) |
| 121 118 | |
| 122 119 | cond do |
| 123 | - count == 0 -> und() |
| 120 | + count == 0 -> |
| 121 | + und() |
| 122 | + |
| 124 123 | Map.has_key?(@languages, script) -> |
| 125 124 | str |
| 126 125 | |> get_clean_trigrams |
| 127 126 | |> get_distances(@languages[script], options) |
| 128 127 | |> normalize(str) |
| 129 128 | |
| 130 | - true -> [{script, 1}] |
| 129 | + true -> |
| 130 | + [{script, 1}] |
| 131 131 | end |
| 132 132 | end |
| 133 133 | |
| 134 134 | defp und, do: [{"und", 1}] |
| 135 135 | |
| 136 | - @spec detect_script(str :: String.t) :: {String.t, number} |
| 136 | + @spec detect_script(str :: String.t()) :: {String.t(), number} |
| 137 137 | defp detect_script(str) do |
| 138 138 | len = String.length(str) |
| 139 139 | |
| @@ -142,12 +142,12 @@ defmodule Paasaa do | |
| 142 142 | |> Enum.max_by(fn {_, count} -> count end) |
| 143 143 | end |
| 144 144 | |
| 145 | - @spec get_occurrence(str :: String.t, re :: Regex.t, str_len :: non_neg_integer) :: float |
| 145 | + @spec get_occurrence(str :: String.t(), re :: Regex.t(), str_len :: non_neg_integer) :: float |
| 146 146 | defp get_occurrence(str, re, str_len) do |
| 147 147 | Enum.count(Regex.scan(re, str)) / str_len |
| 148 148 | end |
| 149 149 | |
| 150 | - @spec get_distances([String.t], Enumerable.t, options) :: result |
| 150 | + @spec get_distances([String.t()], Enumerable.t(), options) :: result |
| 151 151 | defp get_distances(trigrams, languages, options) do |
| 152 152 | languages |
| 153 153 | |> filter_languages(options) |
| @@ -155,18 +155,19 @@ defmodule Paasaa do | |
| 155 155 | |> Enum.sort(&(elem(&1, 1) < elem(&2, 1))) |
| 156 156 | end |
| 157 157 | |
| 158 | - @spec get_distance([String.t], Enumerable.t) :: number |
| 158 | + @spec get_distance([String.t()], Enumerable.t()) :: number |
| 159 159 | defp get_distance(trigrams, model) do |
| 160 | - Enum.reduce trigrams, 0, fn {name, val}, distance -> |
| 161 | - distance + if Map.has_key?(model, name) do |
| 162 | - abs val - model[name] - 1 |
| 163 | - else |
| 164 | - @max_difference |
| 165 | - end |
| 166 | - end |
| 160 | + Enum.reduce(trigrams, 0, fn {name, val}, distance -> |
| 161 | + distance + |
| 162 | + if Map.has_key?(model, name) do |
| 163 | + abs(val - model[name] - 1) |
| 164 | + else |
| 165 | + @max_difference |
| 166 | + end |
| 167 | + end) |
| 167 168 | end |
| 168 169 | |
| 169 | - @spec filter_languages([String.t], Enumerable.t) :: Enumerable.t |
| 170 | + @spec filter_languages([String.t()], Enumerable.t()) :: Enumerable.t() |
| 170 171 | defp filter_languages(languages, options) do |
| 171 172 | white = options[:whitelist] |
| 172 173 | black = options[:blacklist] |
| @@ -174,58 +175,64 @@ defmodule Paasaa do | |
| 174 175 | if Enum.empty?(white) && Enum.empty?(black) do |
| 175 176 | languages |
| 176 177 | else |
| 177 | - Enum.filter languages, fn {lang, _} -> |
| 178 | + Enum.filter(languages, fn {lang, _} -> |
| 178 179 | (Enum.empty?(white) || Enum.member?(white, lang)) && !Enum.member?(black, lang) |
| 179 | - end |
| 180 | + end) |
| 180 181 | end |
| 181 182 | end |
| 182 183 | |
| 183 | - @spec normalize(result, String.t) :: result |
| 184 | + @spec normalize(result, String.t()) :: result |
| 184 185 | defp normalize([], _str), do: und() |
| 186 | + |
| 185 187 | defp normalize(distances, str) do |
| 186 | - min = distances |> List.first |> elem(1) |
| 188 | + min = distances |> List.first() |> elem(1) |
| 187 189 | max = String.length(str) * @max_difference - min |
| 188 190 | |
| 189 | - Enum.map distances, fn({lang, dist}) -> |
| 190 | - dist = if max == 0 do 0 else 1 - ((dist - min) / max) end |
| 191 | + Enum.map(distances, fn {lang, dist} -> |
| 192 | + dist = |
| 193 | + if max == 0 do |
| 194 | + 0 |
| 195 | + else |
| 196 | + 1 - (dist - min) / max |
| 197 | + end |
| 191 198 | |
| 192 199 | {lang, dist} |
| 193 | - end |
| 200 | + end) |
| 194 201 | end |
| 195 202 | |
| 196 203 | # trigram stuff |
| 197 204 | |
| 198 | - @spec get_clean_trigrams(String.t) :: result |
| 205 | + @spec get_clean_trigrams(String.t()) :: result |
| 199 206 | defp get_clean_trigrams(str) do |
| 200 207 | str |
| 201 208 | |> clean |
| 202 209 | |> pad |
| 203 210 | |> n_grams |
| 204 | - |> Enum.reduce(%{}, fn(trigram, acc) -> |
| 205 | - count = acc[trigram] && acc[trigram] + 1 || 1 |
| 206 | - Map.put(acc, trigram, count) |
| 207 | - end) |
| 208 | - |> Map.to_list |
| 211 | + |> Enum.reduce(%{}, fn trigram, acc -> |
| 212 | + count = (acc[trigram] && acc[trigram] + 1) || 1 |
| 213 | + Map.put(acc, trigram, count) |
| 214 | + end) |
| 215 | + |> Map.to_list() |
| 209 216 | end |
| 210 217 | |
| 211 | - @spec clean(str :: String.t) :: String.t |
| 218 | + @spec clean(str :: String.t()) :: String.t() |
| 212 219 | defp clean(str) do |
| 213 220 | expression_symbols = ~r/[\x{0021}-\x{0040}]+/u |
| 214 221 | |
| 215 222 | str |
| 216 223 | |> String.replace(expression_symbols, " ") |
| 217 224 | |> String.replace(~r/\s+/, " ") |
| 218 | - |> String.trim |
| 219 | - |> String.downcase |
| 225 | + |> String.trim() |
| 226 | + |> String.downcase() |
| 220 227 | end |
| 221 228 | |
| 222 229 | defp pad(str), do: " #{str} " |
| 223 230 | |
| 224 | - @spec n_grams(str :: String.t, n :: number) :: [String.t] |
| 231 | + @spec n_grams(str :: String.t(), n :: number) :: [String.t()] |
| 225 232 | defp n_grams(str, n \\ 3) do |
| 226 233 | str |
| 227 | - |> String.graphemes |
| 228 | - |> Enum.chunk(n, 1) |
| 234 | + |> String.graphemes() |
| 235 | + |> Enum.chunk_every(n, 1, :discard) |
| 229 236 | |> Enum.map(&Enum.join/1) |
| 230 237 | end |
| 231 238 | end |
| @@ -4,13 +4,27 @@ defmodule Paasaa.Mixfile do | |
| 4 4 | def project do |
| 5 5 | [ |
| 6 6 | app: :paasaa, |
| 7 | - version: "0.2.2", |
| 8 | - elixir: "~> 1.5", |
| 7 | + version: "0.3.0", |
| 8 | + elixir: "~> 1.7", |
| 9 9 | description: description(), |
| 10 10 | package: package(), |
| 11 11 | deps: deps(), |
| 12 12 | test_coverage: [tool: ExCoveralls], |
| 13 | - preferred_cli_env: ["coveralls": :test, "coveralls.detail": :test, "coveralls.post": :test, "coveralls.html": :test] |
| 13 | + preferred_cli_env: [ |
| 14 | + coveralls: :test, |
| 15 | + "coveralls.detail": :test, |
| 16 | + "coveralls.post": :test, |
| 17 | + "coveralls.html": :test |
| 18 | + ], |
| 19 | + # Docs |
| 20 | + name: "Paasaa", |
| 21 | + source_url: "https://github.com/minibikini/paasaa", |
| 22 | + homepage_url: "https://github.com/minibikini/paasaa", |
| 23 | + docs: [ |
| 24 | + # The main page in the docs |
| 25 | + main: "Paasaa", |
| 26 | + extras: ["README.md"] |
| 27 | + ] |
| 14 28 | ] |
| 15 29 | end |
| 16 30 | |
| @@ -20,11 +34,11 @@ defmodule Paasaa.Mixfile do | |
| 20 34 | |
| 21 35 | defp deps do |
| 22 36 | [ |
| 23 | - {:exjsx, "~> 4.0", only: [:dev, :test]}, |
| 37 | + {:exjsx, "~> 4.0"}, |
| 24 38 | {:ex_doc, "~> 0.12", only: :dev}, |
| 25 39 | {:mix_test_watch, "~> 0.2", only: :dev}, |
| 26 40 | {:benchfella, "~> 0.3.0", only: :dev}, |
| 27 | - {:credo, "~> 0.4", only: [:dev, :test]}, |
| 41 | + {:credo, "~> 1.0.5", only: [:dev, :test]}, |
| 28 42 | {:excoveralls, "~> 0.5", only: :test}, |
| 29 43 | {:dialyxir, "~> 0.5.0", only: [:dev]} |
| 30 44 | ] |
| @@ -45,5 +59,4 @@ defmodule Paasaa.Mixfile do | |
| 45 59 | links: %{"GitHub" => "https://github.com/minibikini/paasaa"} |
| 46 60 | ] |
| 47 61 | end |
| 48 | - |
| 49 62 | end |
unknownpriv/languages.binary
File is too large to be displayed (100 KB limit).
Loading more files…