Packages

paasaa

0.5.0
1.0.0 0.6.0 0.5.1 0.5.0 0.4.0 0.3.1 0.3.0 0.2.2 0.2.1 0.2.0 0.1.1 0.1.0

Natural language detection

HexDocs

Current section

12 Versions

Jump to
Readme
12 Versions
0 Dependencies
Dependants
Files
Activity
Hide whitespace

Compare versions

6 files changed
+46 additions
-25 deletions
  @@ -1,13 +1,13 @@
1 1 # Paasaa
2 2
3 - [![Travis](https://img.shields.io/travis/minibikini/paasaa.svg?maxAge=2592000)](https://travis-ci.org/minibikini/paasaa)
3 + [![Actions Status](https://github.com/minibikini/paasaa/workflows/CI/badge.svg)](https://github.com/minibikini/paasaa/actions)
4 4 [![Coverage Status](https://coveralls.io/repos/github/minibikini/paasaa/badge.svg?branch=master)](https://coveralls.io/github/minibikini/paasaa?branch=master)
5 5 [![Hex.pm](https://img.shields.io/hexpm/v/paasaa.svg?maxAge=2592000)](https://hex.pm/packages/paasaa)
6 6 [![Hex.pm](https://img.shields.io/hexpm/l/paasaa.svg?maxAge=2592000)](https://hex.pm/packages/paasaa)
7 7
8 8 Natural language detection for Elixir
9 9
10 - [Api Documentation] | [Hex Package]
10 + [API Documentation] | [Hex Package]
11 11
12 12 ## Installation
13 13
  @@ -15,7 +15,7 @@ Add `paasaa` to your list of dependencies in `mix.exs`:
15 15
16 16 ```elixir
17 17 def deps do
18 - [{:paasaa, "~> 0.3.0"}]
18 + [{:paasaa, "~> 0.5.0"}]
19 19 end
20 20 ```
21 21
  @@ -43,6 +43,18 @@ iex> Paasaa.all("Detect this!")
43 43 ]
44 44 ```
45 45
46 + ## Benchmark
47 +
48 + ```shell
49 + mix bench
50 + ```
51 +
52 + ## Update Language Data
53 +
54 + ```shell
55 + mix run script/generate_language_data.ex
56 + ```
57 +
46 58 ## Derivation
47 59
48 60 **Paasaa** is a derivative work from [Franc][] (JavaScript, MIT) by [Titus Wormer].
  @@ -1,7 +1,7 @@
1 1 {<<"app">>,<<"paasaa">>}.
2 2 {<<"build_tools">>,[<<"mix">>]}.
3 3 {<<"description">>,<<"Natural language detection">>}.
4 - {<<"elixir">>,<<"~> 1.7">>}.
4 + {<<"elixir">>,<<"~> 1.8">>}.
5 5 {<<"files">>,
6 6 [<<"lib">>,<<"lib/paasaa.ex">>,<<"lib/paasaa">>,<<"lib/paasaa/scripts.ex">>,
7 7 <<"lib/paasaa/languages.ex">>,<<"lib/paasaa/data.ex">>,<<"mix.exs">>,
  @@ -10,4 +10,4 @@
10 10 {<<"links">>,[{<<"GitHub">>,<<"https://github.com/minibikini/paasaa">>}]}.
11 11 {<<"name">>,<<"paasaa">>}.
12 12 {<<"requirements">>,[]}.
13 - {<<"version">>,<<"0.4.0">>}.
13 + {<<"version">>,<<"0.5.0">>}.
  @@ -31,6 +31,8 @@ defmodule Paasaa do
31 31 blacklist: []
32 32 ]
33 33
34 + @und [{"und", 1}]
35 +
34 36 @doc """
35 37 Detects a language. Returns a string with ISO6393 language code (e.g. "eng").
36 38
  @@ -97,14 +99,14 @@ defmodule Paasaa do
97 99
98 100 @spec all(str :: String.t(), options) :: result
99 101 def all(str, options \\ @default_options)
100 - def all("", _), do: und()
101 - def all(nil, _), do: und()
102 + def all("", _), do: @und
103 + def all(nil, _), do: @und
102 104
103 105 def all(str, options) do
104 106 options = Keyword.merge(@default_options, options)
105 107
106 108 if String.length(str) < options[:min_length] do
107 - und()
109 + @und
108 110 else
109 111 process(str, options)
110 112 end
  @@ -118,7 +120,7 @@ defmodule Paasaa do
118 120
119 121 cond do
120 122 count == 0 ->
121 - und()
123 + @und
122 124
123 125 Map.has_key?(@languages, script) ->
124 126 str
  @@ -131,7 +133,18 @@ defmodule Paasaa do
131 133 end
132 134 end
133 135
134 - defp und, do: [{"und", 1}]
136 + @doc """
137 + Detects a script.
138 +
139 + ## Parameters
140 +
141 + - `str` - a text string
142 +
143 + ## Examples
144 +
145 + iex> Paasaa.detect_script("Detect this!")
146 + {"Latin", 0.8333333333333334}
147 + """
135 148
136 149 @spec detect_script(str :: String.t()) :: {String.t(), number}
137 150 def detect_script(str) do
  @@ -182,19 +195,14 @@ defmodule Paasaa do
182 195 end
183 196
184 197 @spec normalize(result, String.t()) :: result
185 - defp normalize([], _str), do: und()
198 + defp normalize([], _str), do: @und
186 199
187 200 defp normalize(distances, str) do
188 201 min = distances |> List.first() |> elem(1)
189 202 max = String.length(str) * @max_difference - min
190 203
191 204 Enum.map(distances, fn {lang, dist} ->
192 - dist =
193 - if max == 0 do
194 - 0
195 - else
196 - 1 - (dist - min) / max
197 - end
205 + dist = if max == 0, do: 0, else: 1 - (dist - min) / max
198 206
199 207 {lang, dist}
200 208 end)
unknownlib/paasaa/languages.ex
File is too large to be displayed (100 KB limit).
  @@ -11,6 +11,7 @@ defmodule Paasaa.Scripts do
11 11 "Hebrew" => "[֑-ׇא-תװ-״יִ-זּטּ-לּמּנּסּףּפּצּ-ﭏ]",
12 12 "Latin" =>
13 13 "[A-Za-z\\xAA\\xBA\\xC0-\\xD6\\xD8-\\xF6\\xF8-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞎꞐ-ꞭꞰꞱꟷ-ꟿꬰ-ꭚꭜ-ꭟꭤff-stＡ-Ｚａ-ｚ]",
14 + "Myanmar" => "[က-႟ꧠ-ꧾꩠ-ꩿ]",
14 15 "aii" => "[܀-܍܏-݊ݍ-ݏ]",
15 16 "ben" => "[ঀ-ঃঅ-ঌএঐও-নপ-রলশ-হ়-ৄেৈো-ৎৗড়ঢ়য়-ৣ০-৻]",
16 17 "bod" => "[ༀ-ཇཉ-ཬཱ-ྗྙ-ྼ྾-࿌࿎-࿔࿙࿚]",
  @@ -20,24 +21,22 @@ defmodule Paasaa.Scripts do
20 21 "guj" => "[ઁ-ઃઅ-ઍએ-ઑઓ-નપ-રલળવ-હ઼-ૅે-ૉો-્ૐૠ-ૣ૦-૱]",
21 22 "hye" => "[Ա-Ֆՙ-՟ա-և֊֍-֏ﬓ-ﬗ]",
22 23 "iii" => "[ꀀ-ꒌ꒐-꓆]",
23 - "jpn" => "[ぁ-ゖゝ-ゟsub]|𛀁|🈀|[ァ-ヺヽ-ヿㇰ-ㇿ㋐-㋾㌀-㍗ｦ-ｯｱ-ﾝ]|𛀀",
24 + "jpn" => "[ぁ-ゖゝ-ゟ]|𛀁|🈀|[ァ-ヺヽ-ヿㇰ-ㇿ㋐-㋾㌀-㍗ｦ-ｯｱ-ﾝ]|𛀀|[㐀-䶵一-龯]",
24 25 "kan" => "[ಁ-ಃಅ-ಌಎ-ಐಒ-ನಪ-ಳವ-ಹ಼-ೄೆ-ೈೊ-್ೕೖೞೠ-ೣ೦-೯ೱೲ]",
25 26 "kat" => "[Ⴀ-ჅჇჍა-ჺჼ-ჿⴀ-ⴥⴧⴭ]",
26 27 "khm" => "[ក-៝០-៩៰-៹᧠-᧿]",
27 28 "kor" => "[ᄀ-ᇿ〮〯ㄱ-ㆎ㈀-㈞㉠-㉾ꥠ-ꥼ가-힣ힰ-ퟆퟋ-ퟻﾠ-ﾾￂ-ￇￊ-ￏￒ-ￗￚ-ￜ]",
28 29 "lao" => "[ກຂຄງຈຊຍດ-ທນ-ຟມ-ຣລວສຫອ-ູົ-ຽເ-ໄໆ່-ໍ໐-໙ໜ-ໟ]",
29 30 "mal" => "[ഁ-ഃഅ-ഌഎ-ഐഒ-ഺഽ-ൄെ-ൈൊ-ൎൗൠ-ൣ൦-൵൹-ൿ]",
30 - "mya" => "[က-႟ꧠ-ꧾꩠ-ꩿ]",
31 31 "ori" => "[ଁ-ଃଅ-ଌଏଐଓ-ନପ-ରଲଳଵ-ହ଼-ୄେୈୋ-୍ୖୗଡ଼ଢ଼ୟ-ୣ୦-୷]",
32 32 "pan" => "[ਁ-ਃਅ-ਊਏਐਓ-ਨਪ-ਰਲਲ਼ਵਸ਼ਸਹ਼ਾ-ੂੇੈੋ-੍ੑਖ਼-ੜਫ਼੦-ੵ]",
33 33 "sat" => "[᱐-᱿]",
34 34 "sin" => "[ංඃඅ-ඖක-නඳ-රලව-ෆ්ා-ුූෘ-ෟ෦-෯ෲ-෴]|�[�-�]",
35 35 "tam" => "[ஂஃஅ-ஊஎ-ஐஒ-கஙசஜஞடணதந-பம-ஹா-ூெ-ைொ-்ௐௗ௦-௺]",
36 36 "tel" => "[ఀ-ఃఅ-ఌఎ-ఐఒ-నప-హఽ-ౄె-ైొ-్ౕౖౘౙౠ-ౣ౦-౯౸-౿]",
37 - "tha" => "[ก-ฺเ-๛]"
37 + "tha" => "[ก-ฺเ-๛]",
38 + "zgh" => "[ⴰ-ⵧⵯ⵰⵿]"
38 39 }
39 40
40 - def get() do
41 - @data
42 - end
41 + def get, do: @data
43 42 end
Loading more files…