Split in sections, sections are split by \n . , ; ! ?.
var tokens = tokenize.sections()('this is sentence 1. this is sentence 2');/*[ { value: 'this is sentence 1', index: 0, offset: 18 }, { value: ' this is sentence 2', index: 19, offset: 19 }]*/
// Filter the words to extract the ones that start with an uppercasevar extractNames = tokenize.filter(function(word, current, prev) { return (prev && /[A-Z]/.test(word[0]));});// Split texts in wordsvar words = tokenize.words()('My name is Samy.');// Apply the filtervar tokens = extractNames(words);/*[ { value: 'Samy', index: 11, offset: 4 }]*/
tokenize.flow(fn1, fn2, [...])
Creates a tokenizer that returns the result of invoking the provided tokenizers for each input token.
var extractNames = tokenize.flow( // Split text as words tokenize.words(), // Filter the words to extract the ones that start with an uppercase tokenize.filter(function(word, current, prev) { return (prev && /[A-Z]/.test(word[0])); }));var tokens = extractNames('My name is Samy.');
To execute all tokenizer in series, you can use tokenize.serie(fn1, fn2, [...]) instead.
Examples
Extract repeated words in sentences
Example to extract all repeated words in sentences:
var repeatedWords = tokenize.flow( // Tokenize as sections tokenize.sections(), // For each sentence tokenize.flow( // Tokenize as words tokenize.words(), // Filter words to extract only repeated ones tokenize.filter(function(word, token, prev) { return ( prev && token.value.toLowerCase() === prev.value.toLowerCase() ); }) ));var tokens = repeatedWords('This is great great. Great is an an awesome words');