A super simple crawling embdder for .md files. Point crawlerjs where your .md files are, give crawlerjs a sentence transformer and watch it go!
Usage:
import Crawler from "./crawler.js";
const { pipeline } = await import("@xenova/transformers");
const crawler = new Crawler(
"data", //directory where .md files are located
"Xenova/all-MiniLM-L6-v2", //sentence transformer
2, //sequence length (look up the sequence model's sequence length)
pipeline); //give it a transformers-js pipeline function
const embeddings = await crawler.crawl();
console.log(embeddings)