import { Firecrawl } from 'firecrawl';import { Document, VectorStoreIndex, Settings } from 'llamaindex';import { OpenAI, OpenAIEmbedding } from '@llamaindex/openai';Settings.llm = new OpenAI({ model: "gpt-4o" });Settings.embedModel = new OpenAIEmbedding({ model: "text-embedding-3-small" });const firecrawl = new Firecrawl({ apiKey: process.env.FIRECRAWL_API_KEY });const crawlResult = await firecrawl.crawl('https://firecrawl.dev', { limit: 10, scrapeOptions: { formats: ['markdown'] }});console.log(`Crawled ${crawlResult.data.length } pages`);const documents = crawlResult.data.map((page: any, i: number) => new Document({ text: page.markdown, id_: `page-${i}`, metadata: { url: page.metadata?.sourceURL } }));const index = await VectorStoreIndex.fromDocuments(documents);console.log('Vector index created with embeddings');const queryEngine = index.asQueryEngine();const response = await queryEngine.query({ query: 'What is Firecrawl and how does it work?' });console.log('\nAnswer:', response.toString());
