Maxim Snesarev
8a8cfcd0f1
- Updated `.gitignore` to exclude scraper intermediate data. - Enhanced `justfile` with new scrape commands for raw data and LLM processing. - Added OpenAI dependency in `scraper/package.json`. - Modified `config.ts` to include OpenRouter API key and model. - Expanded `extract.ts` to capture additional product details such as discount percentage, frost resistance, and size options. - Updated `import.ts` to handle new product attributes during import. - Enhanced `index.ts` to support raw scraping mode and improved product extraction logic. - Updated `payload-types.ts` and `Products.ts` to include new product fields. - Enhanced frontend components to display new product attributes and filters for availability and frost resistance. - Improved error handling and logging in scraper functions. - Added new features for managing product variants and specifications in the admin interface.
30 lines
900 B
TypeScript
30 lines
900 B
TypeScript
import { mkdir, writeFile } from "node:fs/promises";
|
||
import path from "node:path";
|
||
import { fileURLToPath } from "node:url";
|
||
import type { RawProduct } from "./llm/types.js";
|
||
|
||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||
const DATA_DIR = path.resolve(__dirname, "../data/raw");
|
||
|
||
function slugifyPath(text: string): string {
|
||
return text
|
||
.toLowerCase()
|
||
.replace(/[^a-zа-яё0-9]+/gi, "-")
|
||
.replace(/^-+|-+$/g, "")
|
||
.replace(/-+/g, "-");
|
||
}
|
||
|
||
export async function dumpCategory(
|
||
categoryName: string,
|
||
products: RawProduct[],
|
||
): Promise<string> {
|
||
await mkdir(DATA_DIR, { recursive: true });
|
||
|
||
const filename = `${slugifyPath(categoryName)}.json`;
|
||
const filepath = path.join(DATA_DIR, filename);
|
||
|
||
await writeFile(filepath, JSON.stringify(products, null, 2), "utf-8");
|
||
console.log(` Wrote ${products.length} products → ${filepath}`);
|
||
return filepath;
|
||
}
|