Current section
12 Versions
Jump to
Current section
12 Versions
Compare versions
6
files changed
+44
additions
-33
deletions
| @@ -36,9 +36,9 @@ Detect language and return a scored list of languages: | |
| 36 36 | iex> Paasaa.all("Detect this!") |
| 37 37 | [ |
| 38 38 | {"eng", 1.0}, |
| 39 | - {"sco", 0.8668304668304668}, |
| 40 | - {"nob", 0.6054054054054054}, |
| 41 | - {"swe", 0.5921375921375922}, |
| 39 | + {"sco", 0.8675529295913343}, |
| 40 | + {"nob", 0.6065977351058591}, |
| 41 | + {"swe", 0.5923190546528804}, |
| 42 42 | ... |
| 43 43 | ] |
| 44 44 | ``` |
| @@ -52,7 +52,7 @@ mix bench | |
| 52 52 | ## Update Language Data |
| 53 53 | |
| 54 54 | ```shell |
| 55 | - mix run script/generate_language_data.ex |
| 55 | + mix run script/generate_language_data.exs |
| 56 56 | ``` |
| 57 57 | |
| 58 58 | ## Derivation |
| @@ -10,4 +10,4 @@ | |
| 10 10 | {<<"links">>,[{<<"GitHub">>,<<"https://github.com/minibikini/paasaa">>}]}. |
| 11 11 | {<<"name">>,<<"paasaa">>}. |
| 12 12 | {<<"requirements">>,[]}. |
| 13 | - {<<"version">>,<<"0.5.0">>}. |
| 13 | + {<<"version">>,<<"0.5.1">>}. |
| @@ -90,10 +90,10 @@ defmodule Paasaa do | |
| 90 90 | iex> Paasaa.all("Detect this!") |> Enum.take(5) |
| 91 91 | [ |
| 92 92 | {"eng", 1.0}, |
| 93 | - {"sco", 0.8668304668304668}, |
| 94 | - {"nob", 0.6054054054054054}, |
| 95 | - {"swe", 0.5921375921375922}, |
| 96 | - {"nno", 0.5518427518427518} |
| 93 | + {"sco", 0.8675529295913343}, |
| 94 | + {"nob", 0.6065977351058591}, |
| 95 | + {"swe", 0.5923190546528804}, |
| 96 | + {"nno", 0.5534219596258001} |
| 97 97 | ] |
| 98 98 | """ |
| 99 99 | |
| @@ -129,10 +129,20 @@ defmodule Paasaa do | |
| 129 129 | |> normalize(str) |
| 130 130 | |
| 131 131 | true -> |
| 132 | - [{script, 1}] |
| 132 | + if allowed?(script, options) do |
| 133 | + [{script, 1}] |
| 134 | + else |
| 135 | + @und |
| 136 | + end |
| 133 137 | end |
| 134 138 | end |
| 135 139 | |
| 140 | + defp allowed?(lang, options) do |
| 141 | + white = options[:whitelist] |
| 142 | + black = options[:blacklist] |
| 143 | + (Enum.empty?(white) || Enum.member?(white, lang)) && !Enum.member?(black, lang) |
| 144 | + end |
| 145 | + |
| 136 146 | @doc """ |
| 137 147 | Detects a script. |
| 138 148 | |
| @@ -189,7 +199,7 @@ defmodule Paasaa do | |
| 189 199 | languages |
| 190 200 | else |
| 191 201 | Enum.filter(languages, fn {lang, _} -> |
| 192 | - (Enum.empty?(white) || Enum.member?(white, lang)) && !Enum.member?(black, lang) |
| 202 | + allowed?(lang, options) |
| 193 203 | end) |
| 194 204 | end |
| 195 205 | end |
unknownlib/paasaa/languages.ex
File is too large to be displayed (100 KB limit).
| @@ -3,37 +3,38 @@ defmodule Paasaa.Scripts do | |
| 3 3 | |
| 4 4 | @data %{ |
| 5 5 | "Arabic" => |
| 6 | - "[-؆-؋؍-ؚ؞ؠ-ؿف-يٖ-ٟ٪-ٯٱ-ۜ۞-ۿݐ-ݿࢠ-ࢲࣤ-ࣿﭐ-﯁ﯓ-ﴽﵐ-ﶏﶒ-ﷇﷰ-﷽ﹰ-ﹴﹶ-ﻼ]|�[�-�]|�[�-��-������-��-��������-�������������-��-��-��-���-��-��-��-��-���]", |
| 7 | - "Cyrillic" => "[Ѐ-҄҇-ԯᴫᵸⷠ-ⷿꙀ-ꚝꚟ]", |
| 8 | - "Devanagari" => "[ऀ-ॐ॓-ॣ०-ॿ꣠-ꣻ]", |
| 6 | + "[-؆-؋؍-ؚ؞ؠ-ؿف-يٖ-ٯٱ-ۜ۞-ۿݐ-ݿࢠ-ࢴࢶ-ࢽ࣓-ࣣ࣡-ࣿﭐ-﯁ﯓ-ﴽﵐ-ﶏﶒ-ﷇﷰ-﷽ﹰ-ﹴﹶ-ﻼ]|�[�-�]|�[�-��-������-��-��������-�������������-��-��-��-���-��-��-��-��-���]", |
| 7 | + "Cyrillic" => "[Ѐ-҄҇-ԯᲀ-ᲈᴫᵸⷠ-ⷿꙀ-ꚟ︮︯]", |
| 8 | + "Devanagari" => "[ऀ-ॐॕ-ॣ०-ॿ꣠-ꣿ]", |
| 9 9 | "Ethiopic" => |
| 10 10 | "[ሀ-ቈቊ-ቍቐ-ቖቘቚ-ቝበ-ኈኊ-ኍነ-ኰኲ-ኵኸ-ኾዀዂ-ዅወ-ዖዘ-ጐጒ-ጕጘ-ፚ፝-፼ᎀ-᎙ⶀ-ⶖⶠ-ⶦⶨ-ⶮⶰ-ⶶⶸ-ⶾⷀ-ⷆⷈ-ⷎⷐ-ⷖⷘ-ⷞꬁ-ꬆꬉ-ꬎꬑ-ꬖꬠ-ꬦꬨ-ꬮ]", |
| 11 | - "Hebrew" => "[֑-ׇא-תװ-״יִ-זּטּ-לּמּנּסּףּפּצּ-ﭏ]", |
| 11 | + "Hebrew" => "[֑-ׇא-תׯ-״יִ-זּטּ-לּמּנּסּףּפּצּ-ﭏ]", |
| 12 12 | "Latin" => |
| 13 | - "[A-Za-z\\xAA\\xBA\\xC0-\\xD6\\xD8-\\xF6\\xF8-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞎꞐ-ꞭꞰꞱꟷ-ꟿꬰ-ꭚꭜ-ꭟꭤff-stＡ-Ｚａ-ｚ]", |
| 13 | + "[A-Za-z\\xAA\\xBA\\xC0-\\xD6\\xD8-\\xF6\\xF8-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞿꟂ-Ᶎꟷ-ꟿꬰ-ꭚꭜ-ꭤꭦꭧff-stＡ-Ｚａ-ｚ]", |
| 14 14 | "Myanmar" => "[က-႟ꧠ-ꧾꩠ-ꩿ]", |
| 15 | - "aii" => "[܀-܍-݊ݍ-ݏ]", |
| 16 | - "ben" => "[ঀ-ঃঅ-ঌএঐও-নপ-রলশ-হ়-ৄেৈো-ৎৗড়ঢ়য়-ৣ০-৻]", |
| 15 | + "aii" => "[܀-܍-݊ݍ-ݏࡠ-ࡪ]", |
| 16 | + "ben" => "[ঀ-ঃঅ-ঌএঐও-নপ-রলশ-হ়-ৄেৈো-ৎৗড়ঢ়য়-ৣ০-৾]", |
| 17 17 | "bod" => "[ༀ-ཇཉ-ཬཱ-ྗྙ-ྼ྾-࿌࿎-࿔࿙࿚]", |
| 18 | - "cmn" => "[⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〻㐀-䶵一-鿌豈-舘並-龎]|[�-��-�][�-�]|�[�-��-�]|�[�-��-�]|�[�-�]|�[�-�]", |
| 18 | + "cmn" => |
| 19 | + "[⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〻㐀-䶵一-鿯豈-舘並-龎]|[�-��-��-��-�][�-�]|�[�-��-�]|�[�-��-�]|�[�-��-�]|�[�-��-�]|�[�-�]|�[�-�]", |
| 19 20 | "ell" => |
| 20 21 | "[Ͱ-ͳ͵-ͷͺ-ͽͿ΄ΆΈ-ΊΌΎ-ΡΣ-ϡϰ-Ͽᴦ-ᴪᵝ-ᵡᵦ-ᵪᶿἀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ῄῆ-ΐῖ-Ί῝-`ῲ-ῴῶ-῾Ωꭥ]|�[�-��]|�[�-�]", |
| 21 | - "guj" => "[ઁ-ઃઅ-ઍએ-ઑઓ-નપ-રલળવ-હ઼-ૅે-ૉો-્ૐૠ-ૣ૦-૱]", |
| 22 | - "hye" => "[Ա-Ֆՙ-՟ա-և֊֍-֏ﬓ-ﬗ]", |
| 22 | + "guj" => "[ઁ-ઃઅ-ઍએ-ઑઓ-નપ-રલળવ-હ઼-ૅે-ૉો-્ૐૠ-ૣ૦-૱ૹ-૿]", |
| 23 | + "hye" => "[Ա-Ֆՙ-ֈ֊֍-֏ﬓ-ﬗ]", |
| 23 24 | "iii" => "[ꀀ-ꒌ꒐-꓆]", |
| 24 | - "jpn" => "[ぁ-ゖゝ-ゟ]|𛀁|🈀|[ァ-ヺヽ-ヿㇰ-ㇿ㋐-㋾㌀-㍗ｦ-ｯｱ-ﾝ]|𛀀|[㐀-䶵一-龯]", |
| 25 | - "kan" => "[ಁ-ಃಅ-ಌಎ-ಐಒ-ನಪ-ಳವ-ಹ಼-ೄೆ-ೈೊ-್ೕೖೞೠ-ೣ೦-೯ೱೲ]", |
| 26 | - "kat" => "[Ⴀ-ჅჇჍა-ჺჼ-ჿⴀ-ⴥⴧⴭ]", |
| 25 | + "jpn" => "[ぁ-ゖゝ-ゟ]|�[�-��-�]|🈀|[ァ-ヺヽ-ヿㇰ-ㇿ㋐-㋾㌀-㍗ｦ-ｯｱ-ﾝ]|�[��-�]|[㐀-䶵一-龯]", |
| 26 | + "kan" => "[ಀ-ಌಎ-ಐಒ-ನಪ-ಳವ-ಹ಼-ೄೆ-ೈೊ-್ೕೖೞೠ-ೣ೦-೯ೱೲ]", |
| 27 | + "kat" => "[Ⴀ-ჅჇჍა-ჺჼ-ჿᲐ-ᲺᲽ-Ჿⴀ-ⴥⴧⴭ]", |
| 27 28 | "khm" => "[ក-៝០-៩៰-៹᧠-᧿]", |
| 28 29 | "kor" => "[ᄀ-ᇿ〮〯ㄱ-ㆎ㈀-㈞㉠-㉾ꥠ-ꥼ가-힣ힰ-ퟆퟋ-ퟻﾠ-ﾾￂ-ￇￊ-ￏￒ-ￗￚ-ￜ]", |
| 29 | - "lao" => "[ກຂຄງຈຊຍດ-ທນ-ຟມ-ຣລວສຫອ-ູົ-ຽເ-ໄໆ່-ໍ໐-໙ໜ-ໟ]", |
| 30 | - "mal" => "[ഁ-ഃഅ-ഌഎ-ഐഒ-ഺഽ-ൄെ-ൈൊ-ൎൗൠ-ൣ൦-൵൹-ൿ]", |
| 30 | + "lao" => "[ກຂຄຆ-ຊຌ-ຣລວ-ຽເ-ໄໆ່-ໍ໐-໙ໜ-ໟ]", |
| 31 | + "mal" => "[ഀ-ഃഅ-ഌഎ-ഐഒ-ൄെ-ൈൊ-൏ൔ-ൣ൦-ൿ]", |
| 31 32 | "ori" => "[ଁ-ଃଅ-ଌଏଐଓ-ନପ-ରଲଳଵ-ହ଼-ୄେୈୋ-୍ୖୗଡ଼ଢ଼ୟ-ୣ୦-୷]", |
| 32 | - "pan" => "[ਁ-ਃਅ-ਊਏਐਓ-ਨਪ-ਰਲਲ਼ਵਸ਼ਸਹ਼ਾ-ੂੇੈੋ-੍ੑਖ਼-ੜਫ਼੦-ੵ]", |
| 33 | + "pan" => "[ਁ-ਃਅ-ਊਏਐਓ-ਨਪ-ਰਲਲ਼ਵਸ਼ਸਹ਼ਾ-ੂੇੈੋ-੍ੑਖ਼-ੜਫ਼੦-੶]", |
| 33 34 | "sat" => "[᱐-᱿]", |
| 34 35 | "sin" => "[ංඃඅ-ඖක-නඳ-රලව-ෆ්ා-ුූෘ-ෟ෦-෯ෲ-෴]|�[�-�]", |
| 35 | - "tam" => "[ஂஃஅ-ஊஎ-ஐஒ-கஙசஜஞடணதந-பம-ஹா-ூெ-ைொ-்ௐௗ௦-௺]", |
| 36 | - "tel" => "[ఀ-ఃఅ-ఌఎ-ఐఒ-నప-హఽ-ౄె-ైొ-్ౕౖౘౙౠ-ౣ౦-౯౸-౿]", |
| 36 | + "tam" => "[ஂஃஅ-ஊஎ-ஐஒ-கஙசஜஞடணதந-பம-ஹா-ூெ-ைொ-்ௐௗ௦-௺]|�[�-��]", |
| 37 | + "tel" => "[ఀ-ఌఎ-ఐఒ-నప-హఽ-ౄె-ైొ-్ౕౖౘ-ౚౠ-ౣ౦-౯౷-౿]", |
| 37 38 | "tha" => "[ก-ฺเ-๛]", |
| 38 39 | "zgh" => "[ⴰ-ⵧⵯ⵰⵿]" |
| 39 40 | } |
Loading more files…