Packages
ex_keyword_extraction
0.1.0
ex_keyword_extraction is an elixir wrapper for keyword_extraction crate - it includes the following algorithms: TF-IDF, Rake, TextRank
Current section
Files
Jump to
Current section
Files
native/exkeywordextractionnative/src/tokenizer.rs
use keyword_extraction::tokenizer::Tokenizer;
#[rustler::nif(schedule = "DirtyCpu")]
pub fn split_into_words(
text: &str,
stop_words: Vec<String>,
punctation: Vec<String>,
) -> Vec<String> {
Tokenizer::new(text, stop_words.as_slice(), get_punctation(&punctation)).split_into_words()
}
#[rustler::nif(schedule = "DirtyCpu")]
pub fn split_into_sentences(
text: &str,
stop_words: Vec<String>,
punctation: Vec<String>,
) -> Vec<String> {
Tokenizer::new(text, stop_words.as_slice(), get_punctation(&punctation)).split_into_sentences()
}
#[rustler::nif(schedule = "DirtyCpu")]
pub fn split_into_paragraphs(
text: &str,
stop_words: Vec<String>,
punctation: Vec<String>,
) -> Vec<String> {
Tokenizer::new(text, stop_words.as_slice(), get_punctation(&punctation)).split_into_paragraphs()
}
#[rustler::nif(schedule = "DirtyCpu")]
pub fn split_into_phrases(
text: &str,
stop_words: Vec<String>,
punctation: Vec<String>,
) -> Vec<String> {
Tokenizer::new(text, stop_words.as_slice(), get_punctation(&punctation))
.split_into_phrases(None)
}
fn get_punctation(punctation: &Vec<String>) -> Option<&[String]> {
match punctation {
v if v.is_empty() => None,
_ => Some(punctation.as_slice()),
}
}