tokenizer
Library to find urls, mentions, hashtags and more.
installation
npm i @tinkoff/tokenizer
API
type Token = { type: string; value: string };
// to specify custom tokens
// you can pass optional param with your own RegExp (as string only)
// ⚠️IMPORTANT! Do not use capturing groups. Use non-capturing groups "(?:)" instead
type Params = { [token: string]: string }; // e.g. { hashtag: '#[a-z]+' }
function tokenizer(str: string, tokensParam: Params[]): Token[];
examples
import { tokenizer, print } from '@tinkoff/tokenizer';
const textToParse = `
text containing some links like example.com or
more strict links with protocol https://google.com;
maybe some unicode written urls like сайт.рф К)
or even a link with hash vk.com/#antient/route
also you can parse custom tokens like {$MAC} and {$CHEESE}
it's easy to mention @someone and get it parsed as well
at last but not least #hashtags should also work fine.
`;
const result = tokenizer(textToParse, {
tag: '\\{\\$MAC\\}|\\{\\$CHEESE\\}',
hashtag: '#[a-z]+',
user: '[@][a-z]+'
})
console.log(print(result));
outputs:
"
text containing some links like " (text) "example.com" (domain) " or
more strict links with protocol " (text) "https://google.com" (domain) ";
maybe some unicode written urls like " (text) "сайт.рф" (domain) " К)
or even a link with hash " (text) "vk.com/#antient/route" (domain) "
also you can parse custom tokens like " (text) "{$MAC}" (tag) " and " (text) "{$CHEESE}" (tag) "
it's easy to mention " (text) "@someone" (user) " and get it parsed as well
at last but not least " (text) "#hashtags" (hashtag) " should also work fine.
" (text)