nicholasandnicholas
2a1e16ed30
Reviewed-on: #14 Co-authored-by: Nicholas Ward <nicholaspward@outlook.com>
796 lines
30 KiB
JavaScript
796 lines
30 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Advanced Docling Recipe Ingestor for "The Pastry Chef's Little Black Book, Vol. I"
|
|
*
|
|
* Supports:
|
|
* - 2-page facing spreads (Table on Left, Procedure on Right)
|
|
* - Multi-component formulation splitting (Dough Packet, Butter Packet, Filling, Crust)
|
|
* - Fractional spoon & unit fallbacks across all columns
|
|
* - Multi-stage procedure preservation with equipment inference
|
|
* - Shelf-life & chef's notes extraction
|
|
*
|
|
* Usage:
|
|
* node scripts/ingest-docling-book.mjs --dry-run --pages 26-27
|
|
* node scripts/ingest-docling-book.mjs --save
|
|
*/
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { DatabaseSync } from "node:sqlite";
|
|
import { titleCase } from "../src/lib/format.ts";
|
|
|
|
const root = path.resolve(import.meta.dirname, "..");
|
|
const databasePath = path.join(root, "var", "recipe-book.sqlite");
|
|
const doclingPath = path.join(root, "file.json");
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Table of Contents Chapter Ranges
|
|
// ---------------------------------------------------------------------------
|
|
export const CHAPTER_PAGE_RANGES = [
|
|
{ name: "Doughs", startPage: 11, endPage: 38, category: "doughs" },
|
|
{ name: "Tart, Pie & Strudel Fillings", startPage: 39, endPage: 64, category: "tart_pie_fillings" },
|
|
{ name: "Cakes & Souffles", startPage: 65, endPage: 122, category: "cakes_souffles" },
|
|
{ name: "Sheet Cakes", startPage: 123, endPage: 166, category: "sheet_cakes" },
|
|
{ name: "Buttercreams, Frostings & Glazes", startPage: 167, endPage: 190, category: "frostings_glazes" },
|
|
{ name: "Custards, Creams & Fillings", startPage: 191, endPage: 234, category: "custards_creams" },
|
|
{ name: "Mousses & Bavarian Creams", startPage: 235, endPage: 296, category: "mousses_bavarians" },
|
|
{ name: "Cookies & Tuiles", startPage: 297, endPage: 350, category: "cookies_tuiles" },
|
|
{ name: "Sauces & Poaching Liquids", startPage: 351, endPage: 380, category: "sauces_liquids" },
|
|
{ name: "Chocolates & Confections", startPage: 381, endPage: 424, category: "confections" },
|
|
{ name: "Frozen Desserts", startPage: 425, endPage: 472, category: "frozen_desserts" },
|
|
{ name: "Breakfast", startPage: 473, endPage: 516, category: "breakfast" },
|
|
{ name: "Breads", startPage: 517, endPage: 537, category: "breads" },
|
|
];
|
|
|
|
export function getChapterForPage(pageNo) {
|
|
for (const ch of CHAPTER_PAGE_RANGES) {
|
|
if (pageNo >= ch.startPage && pageNo <= ch.endPage) return ch;
|
|
}
|
|
return { name: "General Pastry", category: "pastry" };
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Extended Ingredient Normalization Map
|
|
// ---------------------------------------------------------------------------
|
|
const KNOWN_INGREDIENT_MAP = {
|
|
"butter": "butter",
|
|
"unsalted butter": "butter",
|
|
"salted butter": "butter_salted",
|
|
"clarified butter": "clarified_butter",
|
|
"brown butter": "brown_butter",
|
|
"beurre noisette": "brown_butter",
|
|
"granulated sugar": "sugar",
|
|
"sugar": "sugar",
|
|
"powdered sugar": "confectioners_sugar",
|
|
"confectioners sugar": "confectioners_sugar",
|
|
"icing sugar": "confectioners_sugar",
|
|
"brown sugar": "brown_sugar",
|
|
"light brown sugar": "brown_sugar",
|
|
"dark brown sugar": "brown_sugar",
|
|
"all-purpose flour": "flour_all_purpose",
|
|
"all purpose flour": "flour_all_purpose",
|
|
"ap flour": "flour_all_purpose",
|
|
"pastry flour": "flour_pastry",
|
|
"cake flour": "flour_cake",
|
|
"bread flour": "flour_bread",
|
|
"fine whole wheat flour": "flour_whole_wheat",
|
|
"whole wheat flour": "flour_whole_wheat",
|
|
"almond flour": "almond_flour",
|
|
"hazelnut flour": "hazelnut_flour",
|
|
"whole eggs": "egg_whole",
|
|
"eggs": "egg_whole",
|
|
"whole egg": "egg_whole",
|
|
"egg yolks": "egg_yolk",
|
|
"egg yolk": "egg_yolk",
|
|
"egg whites": "egg_whites",
|
|
"egg white": "egg_whites",
|
|
"whole milk": "milk_whole",
|
|
"milk": "milk_whole",
|
|
"milk powder": "milk_powder",
|
|
"nonfat dry milk": "milk_powder",
|
|
"heavy cream": "heavy_cream",
|
|
"cream": "heavy_cream",
|
|
"heavy cream 36%": "heavy_cream",
|
|
"heavy cream 40%": "heavy_cream",
|
|
"sour cream": "sour_cream",
|
|
"creme fraiche": "sour_cream",
|
|
"mascarpone": "mascarpone",
|
|
"cream cheese": "cream_cheese",
|
|
"buttermilk": "buttermilk",
|
|
"salt": "salt",
|
|
"fine salt": "salt",
|
|
"kosher salt": "salt",
|
|
"sea salt": "salt",
|
|
"baking powder": "baking_powder",
|
|
"baking soda": "baking_soda",
|
|
"cream of tartar": "cream_of_tartar",
|
|
"vanilla extract": "vanilla_extract",
|
|
"vanilla bean": "vanilla_bean",
|
|
"vanilla beans": "vanilla_bean",
|
|
"vanilla paste": "vanilla_extract",
|
|
"almond extract": "almond_extract",
|
|
"cinnamon (ground)": "cinnamon",
|
|
"cinnamon": "cinnamon",
|
|
"ground cinnamon": "cinnamon",
|
|
"nutmeg": "nutmeg",
|
|
"ground nutmeg": "nutmeg",
|
|
"black pepper": "black_pepper",
|
|
"white vinegar": "white_vinegar",
|
|
"vinegar": "white_vinegar",
|
|
"water": "water",
|
|
"water (cold)": "water",
|
|
"water (warm)": "water",
|
|
"water (hot)": "water",
|
|
"cocoa powder": "cocoa_powder",
|
|
"dutch-process cocoa powder": "cocoa_powder",
|
|
"cocoa butter": "cocoa_butter",
|
|
"dark chocolate": "chocolate_dark",
|
|
"chocolate": "chocolate_dark",
|
|
"dark chocolate 64%": "chocolate_dark",
|
|
"dark chocolate 70%": "chocolate_dark",
|
|
"semisweet chocolate": "chocolate_dark",
|
|
"bittersweet chocolate": "chocolate_dark",
|
|
"milk chocolate": "chocolate_milk",
|
|
"white chocolate": "chocolate_white",
|
|
"cornstarch": "cornstarch",
|
|
"gelatin (sheet)": "gelatin_sheet",
|
|
"gelatin (powder)": "gelatin_powder",
|
|
"gelatin sheets": "gelatin_sheet",
|
|
"sheet gelatin": "gelatin_sheet",
|
|
"powdered gelatin": "gelatin_powder",
|
|
"honey": "honey",
|
|
"glucose syrup": "glucose_syrup",
|
|
"glucose": "glucose_syrup",
|
|
"powdered glucose": "powdered_glucose",
|
|
"corn syrup": "corn_syrup",
|
|
"trimoline": "invert_sugar",
|
|
"invert sugar": "invert_sugar",
|
|
"canola oil": "canola_oil",
|
|
"vegetable oil": "canola_oil",
|
|
"olive oil": "olive_oil",
|
|
"lemon juice": "lemon_juice",
|
|
"lemon zest": "lemon_zest",
|
|
"lemon or lime zest": "lemon_zest",
|
|
"lime zest": "lime_zest",
|
|
"lemons": "lemon",
|
|
"orange juice": "orange_juice",
|
|
"orange zest": "orange_zest",
|
|
"lime juice": "lime_juice",
|
|
"loose tea": "tea_loose",
|
|
"chopped nuts": "walnut",
|
|
"passion fruit puree": "passion_fruit_puree",
|
|
"raspberry puree": "raspberry_puree",
|
|
"strawberry puree": "strawberry_puree",
|
|
"mango puree": "mango_puree",
|
|
"almond paste": "almond_paste",
|
|
"marzipan": "marzipan",
|
|
"praline paste": "praline_paste",
|
|
"hazelnut paste": "hazelnut_paste",
|
|
"pistachio paste": "pistachio_paste",
|
|
"walnuts": "walnut",
|
|
"walnut": "walnut",
|
|
"pecans": "pecan",
|
|
"almonds": "almond",
|
|
"hazelnuts": "hazelnut",
|
|
"pistachios": "pistachio",
|
|
"fresh yeast": "yeast_fresh",
|
|
"yeast (fresh)": "yeast_fresh",
|
|
"instant yeast": "yeast_instant",
|
|
"yeast (instant)": "yeast_instant",
|
|
"active dry yeast": "yeast_active_dry",
|
|
"ice cream stabilizer": "ice_cream_stabilizer",
|
|
"sorbet stabilizer": "sorbet_stabilizer",
|
|
"pectin nh": "pectin_nh",
|
|
"pectin yellow": "pectin_yellow",
|
|
"pectin": "pectin",
|
|
};
|
|
|
|
function slugify(text) {
|
|
return text
|
|
.toLowerCase()
|
|
.normalize("NFKD")
|
|
.replace(/[\u0300-\u036f]/g, "")
|
|
.replace(/[^a-z0-9]+/g, "_")
|
|
.replace(/^_+|_+$/g, "") || "recipe";
|
|
}
|
|
|
|
function parseMetricAmount(str) {
|
|
if (!str) return null;
|
|
const s = str.trim().toLowerCase();
|
|
|
|
const kgMatch = s.match(/^([\d.,]+)\s*kg$/i);
|
|
if (kgMatch) {
|
|
return { quantity: Math.round(parseFloat(kgMatch[1].replace(/,/g, "")) * 1000 * 100) / 100, unit_id: "gram" };
|
|
}
|
|
|
|
const gMatch = s.match(/^([\d.,]+)\s*g$/i);
|
|
if (gMatch) {
|
|
return { quantity: Math.round(parseFloat(gMatch[1].replace(/,/g, "")) * 100) / 100, unit_id: "gram" };
|
|
}
|
|
|
|
const mlMatch = s.match(/^([\d.,]+)\s*ml$/i);
|
|
if (mlMatch) {
|
|
return { quantity: Math.round(parseFloat(mlMatch[1].replace(/,/g, "")) * 100) / 100, unit_id: "milliliter" };
|
|
}
|
|
|
|
const lMatch = s.match(/^([\d.,]+)\s*l$/i);
|
|
if (lMatch) {
|
|
return { quantity: Math.round(parseFloat(lMatch[1].replace(/,/g, "")) * 1000 * 100) / 100, unit_id: "milliliter" };
|
|
}
|
|
|
|
const numMatch = s.match(/^([\d.,]+)$/);
|
|
if (numMatch) {
|
|
return { quantity: Math.round(parseFloat(numMatch[1].replace(/,/g, "")) * 100) / 100, unit_id: "gram" };
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
export function cleanFractionText(str) {
|
|
if (!str) return "";
|
|
return str
|
|
.replace(/[\r\n]+/g, " ")
|
|
.replace(/(\d+)\s*\/\s*\1\s*\/\s*(\d+)/g, (m, a, b) => `${a}/${b}`)
|
|
.replace(/(\d+)\s*\/\s*(\d+)/g, (m, a, b) => `${a}/${b}`)
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
}
|
|
|
|
function parseUsFallback(text, name) {
|
|
if (!text) {
|
|
if (/zest/i.test(name)) return { quantity: 1, unit_id: "each", notes: "Zest of 1" };
|
|
if (/vanilla bean/i.test(name)) return { quantity: 1, unit_id: "each", notes: "1 bean" };
|
|
return { quantity: 1, unit_id: "gram", notes: "To taste / as needed" };
|
|
}
|
|
const s = cleanFractionText(text).toLowerCase();
|
|
if (s.includes("1/8") || s.includes("⅛")) return { quantity: 0.6, unit_id: "gram", notes: "⅛ tsp" };
|
|
if (s.includes("1/4") || s.includes("¼")) return { quantity: 1.25, unit_id: "gram", notes: "¼ tsp" };
|
|
if (s.includes("1/2") || s.includes("½")) return { quantity: 2.5, unit_id: "gram", notes: "½ tsp" };
|
|
if (s.includes("3/4") || s.includes("¾")) return { quantity: 3.75, unit_id: "gram", notes: "¾ tsp" };
|
|
if (s.includes("1 1/4") || s.includes("1¼")) return { quantity: 6.25, unit_id: "gram", notes: "1¼ tsp" };
|
|
if (s.includes("1 1/2") || s.includes("1½")) return { quantity: 7.5, unit_id: "gram", notes: "1½ tsp" };
|
|
if (s.includes("2 t")) return { quantity: 10, unit_id: "gram", notes: "2 tsp" };
|
|
if (s.includes("1 t") && !s.includes("tbsp")) return { quantity: 5, unit_id: "gram", notes: "1 tsp" };
|
|
if (s.includes("tbsp") || s.includes("1 t") || s.includes("2 t")) return { quantity: 15, unit_id: "gram", notes: "1 Tbsp" };
|
|
|
|
const eachMatch = s.match(/^([\d.]+)\s*(?:each|pc|ea)?$/);
|
|
if (eachMatch && parseFloat(eachMatch[1]) > 0) return { quantity: parseFloat(eachMatch[1]), unit_id: "each", notes: null };
|
|
|
|
const ozMatch = s.match(/^([\d.]+)\s*oz$/);
|
|
if (ozMatch && parseFloat(ozMatch[1]) > 0) {
|
|
return { quantity: Math.round(parseFloat(ozMatch[1]) * 28.3495 * 100) / 100, unit_id: "gram", notes: cleanFractionText(text) };
|
|
}
|
|
|
|
return { quantity: 1, unit_id: "gram", notes: cleanFractionText(text) };
|
|
}
|
|
|
|
function cleanIngredientName(raw) {
|
|
let cleaned = cleanFractionText(raw).replace(/^[\s•\-\*]+/, "");
|
|
let notes = null;
|
|
|
|
const parenMatch = cleaned.match(/^([^(]+)\s*\(([^)]+)\)$/);
|
|
if (parenMatch) {
|
|
const baseName = parenMatch[1].trim();
|
|
const parenContent = parenMatch[2].trim();
|
|
|
|
if (/streusel/i.test(baseName)) {
|
|
if (/zest/i.test(parenContent)) {
|
|
const fruit = baseName.replace(/streusel/i, "").trim();
|
|
cleaned = `${fruit} Zest`;
|
|
notes = `Zest of whole fruit (for ${baseName})`;
|
|
} else if (/loose tea/i.test(parenContent)) {
|
|
cleaned = "Loose Tea";
|
|
notes = `For ${baseName}`;
|
|
} else if (/chopped/i.test(parenContent)) {
|
|
const nutType = baseName.replace(/streusel/i, "").trim();
|
|
cleaned = /nut/i.test(nutType) ? "Chopped Nuts" : (nutType || "Nuts");
|
|
notes = `${parenContent} (for ${baseName})`;
|
|
} else {
|
|
cleaned = baseName;
|
|
notes = parenContent;
|
|
}
|
|
} else {
|
|
cleaned = baseName;
|
|
notes = parenContent;
|
|
}
|
|
}
|
|
|
|
return { name: cleaned, notes };
|
|
}
|
|
|
|
function inferIngredientId(name) {
|
|
const norm = name.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
|
|
if (KNOWN_INGREDIENT_MAP[norm]) return KNOWN_INGREDIENT_MAP[norm];
|
|
return slugify(norm);
|
|
}
|
|
|
|
function isFlourBasis(ingredientId, name) {
|
|
const n = (ingredientId + " " + name).toLowerCase();
|
|
return (
|
|
n.includes("flour") &&
|
|
!n.includes("almond") &&
|
|
!n.includes("hazelnut") &&
|
|
!n.includes("cornstarch")
|
|
);
|
|
}
|
|
|
|
function inferEquipment(instruction) {
|
|
const text = instruction.toLowerCase();
|
|
const eq = new Set();
|
|
if (text.includes("mixer") || text.includes("paddle") || text.includes("whip") || text.includes("dough hook")) eq.add("stand_mixer");
|
|
if (text.includes("whisk")) eq.add("whisk");
|
|
if (text.includes("bowl")) eq.add("mixing_bowl");
|
|
if (text.includes("scale") || text.includes("weigh")) eq.add("kitchen_scale");
|
|
if (text.includes("bake") || text.includes("oven") || text.includes("375°f") || text.includes("350°f") || text.includes("325°f")) eq.add("oven");
|
|
if (text.includes("sheet pan") || text.includes("parchment") || text.includes("silpat")) eq.add("sheet_pan");
|
|
if (text.includes("saucepan") || text.includes("simmer") || text.includes("boil") || text.includes("pot")) eq.add("saucepan");
|
|
if (text.includes("food processor") || text.includes("process") || text.includes("robot coupe")) eq.add("food_processor");
|
|
if (text.includes("blender") || text.includes("blend") || text.includes("immersion blender")) eq.add("blender");
|
|
if (text.includes("thermometer") || text.includes("degrees") || text.includes("°c") || text.includes("°f")) eq.add("thermometer");
|
|
return [...eq];
|
|
}
|
|
|
|
function parseShelfLife(notesList) {
|
|
for (const note of notesList) {
|
|
const text = note.toLowerCase();
|
|
const dayMatch = text.match(/refrigerat\w*\s+for\s+(\d+)\s+days?/i);
|
|
if (dayMatch) {
|
|
return {
|
|
quantity: parseInt(dayMatch[1], 10),
|
|
unit: "day",
|
|
storage_condition: "refrigerated",
|
|
};
|
|
}
|
|
const monthMatch = text.match(/frozen\s+(?:up\s+to\s+)?(\d+)\s+months?/i);
|
|
if (monthMatch) {
|
|
return {
|
|
quantity: parseInt(monthMatch[1], 10) * 30,
|
|
unit: "day",
|
|
storage_condition: "frozen",
|
|
};
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Page & Recipe Extractor
|
|
// ---------------------------------------------------------------------------
|
|
export function parseDoclingBook(doclingJson, targetPages = null) {
|
|
const pagesMap = new Map();
|
|
|
|
for (const textNode of doclingJson.texts || []) {
|
|
const pageNo = textNode.prov?.[0]?.page_no;
|
|
if (!pageNo) continue;
|
|
if (targetPages && !targetPages.includes(pageNo)) continue;
|
|
|
|
if (!pagesMap.has(pageNo)) pagesMap.set(pageNo, { pageNo, texts: [], tables: [] });
|
|
pagesMap.get(pageNo).texts.push(textNode);
|
|
}
|
|
|
|
for (const tableNode of doclingJson.tables || []) {
|
|
const pageNo = tableNode.prov?.[0]?.page_no;
|
|
if (!pageNo) continue;
|
|
if (targetPages && !targetPages.includes(pageNo)) continue;
|
|
|
|
if (!pagesMap.has(pageNo)) pagesMap.set(pageNo, { pageNo, texts: [], tables: [] });
|
|
pagesMap.get(pageNo).tables.push(tableNode);
|
|
}
|
|
|
|
const recipes = [];
|
|
const usedSlugs = new Map();
|
|
const sortedPages = [...pagesMap.keys()].sort((a, b) => a - b);
|
|
|
|
for (const pageNo of sortedPages) {
|
|
const page = pagesMap.get(pageNo);
|
|
const chapterInfo = getChapterForPage(pageNo);
|
|
|
|
if (page.tables.length === 0) continue;
|
|
|
|
for (const table of page.tables) {
|
|
const cells = table.data?.table_cells || [];
|
|
if (cells.length < 4) continue;
|
|
|
|
const grid = new Map();
|
|
let maxRow = 0;
|
|
let maxCol = 0;
|
|
for (const cell of cells) {
|
|
const r = cell.start_row_offset_idx;
|
|
const c = cell.start_col_offset_idx;
|
|
if (!grid.has(r)) grid.set(r, new Map());
|
|
grid.get(r).set(c, cell.text?.trim() || "");
|
|
if (r > maxRow) maxRow = r;
|
|
if (c > maxCol) maxCol = c;
|
|
}
|
|
|
|
const headerRow = grid.get(0);
|
|
const isIngredientTable = headerRow && [...headerRow.values()].some((v) => /ingredients/i.test(v));
|
|
if (!isIngredientTable) continue;
|
|
|
|
// Find Recipe Title on this page
|
|
const titleNode = page.texts.find(
|
|
(t) => t.label === "section_header" && !/procedure|chef's notes|notes|table of contents|scaling|baking/i.test(t.text)
|
|
);
|
|
const title = titleNode ? titleNode.text.trim() : `Recipe Page ${pageNo}`;
|
|
let slugId = slugify(title);
|
|
if (usedSlugs.has(slugId)) {
|
|
const count = usedSlugs.get(slugId) + 1;
|
|
usedSlugs.set(slugId, count);
|
|
slugId = `${slugId}_p${pageNo}`;
|
|
} else {
|
|
usedSlugs.set(slugId, 1);
|
|
}
|
|
|
|
// Parse ingredients into components
|
|
const components = [];
|
|
let currentComponent = { id: "main", name: "Main", items: [] };
|
|
components.push(currentComponent);
|
|
|
|
let totalYieldGrams = null;
|
|
|
|
for (let r = 1; r <= maxRow; r++) {
|
|
const row = grid.get(r);
|
|
if (!row) continue;
|
|
|
|
const ingText = cleanFractionText(row.get(0) || "");
|
|
const metricText = cleanFractionText(row.get(1) || "");
|
|
const usText = cleanFractionText(row.get(2) || "");
|
|
|
|
if (/total weight/i.test(ingText)) {
|
|
const parsedTotal = parseMetricAmount(metricText) || parseMetricAmount(usText);
|
|
if (parsedTotal) totalYieldGrams = parsedTotal.quantity;
|
|
continue;
|
|
}
|
|
|
|
if (!ingText) continue;
|
|
|
|
// Detect sub-component headers inside tables like "Dough Packet (Détrempe):" or "Filling:"
|
|
if (ingText.endsWith(":") && !metricText && !usText) {
|
|
const compName = ingText.replace(/:$/, "").trim();
|
|
const compSlug = slugify(compName);
|
|
if (currentComponent.items.length === 0 && components.length === 1) {
|
|
currentComponent.id = compSlug;
|
|
currentComponent.name = compName;
|
|
} else {
|
|
currentComponent = { id: compSlug, name: compName, items: [] };
|
|
components.push(currentComponent);
|
|
}
|
|
continue;
|
|
}
|
|
|
|
const { name, notes: parenNotes } = cleanIngredientName(ingText);
|
|
const ingredientId = inferIngredientId(name);
|
|
|
|
let parsedMetric = parseMetricAmount(metricText);
|
|
let notes = parenNotes;
|
|
let quantity = parsedMetric ? parsedMetric.quantity : 0;
|
|
let unitId = parsedMetric ? parsedMetric.unit_id : "gram";
|
|
|
|
if (quantity <= 0) {
|
|
const fallback = parseUsFallback(metricText || usText, name);
|
|
quantity = fallback.quantity;
|
|
unitId = fallback.unit_id;
|
|
if (fallback.notes) {
|
|
notes = notes ? `${notes} (${fallback.notes})` : fallback.notes;
|
|
}
|
|
} else if (usText && !notes) {
|
|
if (/[½¼¾t]/i.test(usText)) {
|
|
notes = usText;
|
|
}
|
|
}
|
|
|
|
currentComponent.items.push({
|
|
raw_name: ingText,
|
|
clean_name: name,
|
|
ingredient_id: ingredientId,
|
|
quantity,
|
|
unit_id: unitId,
|
|
us_measure: usText,
|
|
notes: notes || null,
|
|
basis_member: isFlourBasis(ingredientId, name),
|
|
});
|
|
}
|
|
|
|
// Filter out empty components
|
|
const validComponents = components.filter((c) => c.items.length > 0);
|
|
if (validComponents.length === 0) continue;
|
|
|
|
// Calculate Baker's Percentages across all components
|
|
const allItems = validComponents.flatMap((c) => c.items);
|
|
const flourBasisWeight = allItems
|
|
.filter((i) => i.basis_member)
|
|
.reduce((sum, i) => sum + i.quantity, 0);
|
|
|
|
let itemCounter = 1;
|
|
const formattedComponents = validComponents.map((comp) => ({
|
|
id: comp.id,
|
|
name: comp.name,
|
|
notes: [],
|
|
items: comp.items.map((item) => {
|
|
let pct = null;
|
|
if (flourBasisWeight > 0 && item.quantity > 0) {
|
|
pct = Number(((item.quantity / flourBasisWeight) * 100).toFixed(2));
|
|
}
|
|
return {
|
|
id: `line_${String(itemCounter++).padStart(2, "0")}_${item.ingredient_id}`,
|
|
ingredient_id: item.ingredient_id,
|
|
name: titleCase(item.clean_name),
|
|
quantity: item.quantity,
|
|
unit_id: item.unit_id,
|
|
percentage: pct,
|
|
basis_member: item.basis_member,
|
|
notes: item.notes,
|
|
};
|
|
}),
|
|
}));
|
|
|
|
// Find procedure steps on current page or facing spread page (pageNo + 1)
|
|
const textSources = [...page.texts];
|
|
const nextPage = pagesMap.get(pageNo + 1);
|
|
if (nextPage && nextPage.tables.length === 0) {
|
|
textSources.push(...nextPage.texts);
|
|
}
|
|
|
|
const WORD_TO_NUMBER = {
|
|
"one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, "dozen": 12, "half": 0.5
|
|
};
|
|
|
|
function parseYieldServings(yieldText) {
|
|
if (!yieldText) return null;
|
|
const s = yieldText.replace(/^yields?:\s*/i, "").trim().toLowerCase();
|
|
const digitMatch = s.match(/^(\d+)/);
|
|
if (digitMatch) return parseInt(digitMatch[1], 10);
|
|
const wordMatch = s.match(/^(one|two|three|four|five|six|seven|eight|nine|ten|dozen|half)/);
|
|
if (wordMatch && WORD_TO_NUMBER[wordMatch[1]]) return WORD_TO_NUMBER[wordMatch[1]];
|
|
return null;
|
|
}
|
|
|
|
function isProcedureHeader(text, label, recipeTitle) {
|
|
if (label !== "section_header") return false;
|
|
const s = text.trim();
|
|
if (recipeTitle && s.toLowerCase() === recipeTitle.toLowerCase()) return false;
|
|
if (/^chef's notes?:?$/i.test(s)) return false;
|
|
if (/^table of contents|foreword|introduction|acknowledgements|how to use/i.test(s)) return false;
|
|
return /procedure|assembly|variations?|baking|shaping|finishing|glaz|infusion/i.test(s) || s.endsWith(":");
|
|
}
|
|
|
|
function cleanProcedureHeader(raw) {
|
|
const t = raw.trim().replace(/^['"]+|['":]+$/g, "").trim();
|
|
if (/^procedure$/i.test(t)) return "";
|
|
const cleaned = t.replace(/\s+procedure$/i, "").trim();
|
|
return cleaned ? `${cleaned}:` : "";
|
|
}
|
|
|
|
const steps = [];
|
|
let inProcedure = false;
|
|
let inChefNotes = false;
|
|
let yieldDescription = null;
|
|
const chefNotesList = [];
|
|
const narrativeTexts = [];
|
|
|
|
for (const textNode of textSources) {
|
|
const text = textNode.text.trim();
|
|
|
|
if (/^yields?:\s*/i.test(text)) {
|
|
yieldDescription = text;
|
|
continue;
|
|
}
|
|
|
|
if (/^chef's notes?:?$/i.test(text)) {
|
|
inProcedure = false;
|
|
inChefNotes = true;
|
|
continue;
|
|
}
|
|
|
|
if (isProcedureHeader(text, textNode.label, title)) {
|
|
inProcedure = true;
|
|
inChefNotes = false;
|
|
const heading = cleanProcedureHeader(text);
|
|
if (heading) {
|
|
steps.push({
|
|
id: `step_${steps.length + 1}`,
|
|
order: steps.length + 1,
|
|
instruction: heading,
|
|
equipment_ids: [],
|
|
});
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (inProcedure) {
|
|
if ((textNode.label === "list_item" || textNode.label === "text") && !/^\d+$/.test(text) && !/^yields?:/i.test(text)) {
|
|
steps.push({
|
|
id: `step_${steps.length + 1}`,
|
|
order: steps.length + 1,
|
|
instruction: text,
|
|
equipment_ids: inferEquipment(text),
|
|
});
|
|
}
|
|
} else if (inChefNotes) {
|
|
if (textNode.label === "list_item") {
|
|
chefNotesList.push(text);
|
|
} else if (textNode.label === "text" && !/^\d+$/.test(text)) {
|
|
narrativeTexts.push(text);
|
|
}
|
|
}
|
|
}
|
|
|
|
const sumWeight = allItems.reduce((s, i) => s + (i.unit_id === "gram" ? i.quantity : 0), 0);
|
|
const yieldQuantity = totalYieldGrams || (sumWeight > 0 ? sumWeight : 1000);
|
|
const yieldServings = parseYieldServings(yieldDescription);
|
|
|
|
const allNotes = [...chefNotesList];
|
|
if (yieldDescription) {
|
|
allNotes.unshift(yieldDescription);
|
|
}
|
|
|
|
const summary = narrativeTexts.length > 0
|
|
? (yieldDescription ? `${yieldDescription}. ${narrativeTexts.join(" ")}` : narrativeTexts.join(" "))
|
|
: yieldDescription || null;
|
|
|
|
recipes.push({
|
|
id: slugId,
|
|
title,
|
|
page_no: pageNo,
|
|
chapter: chapterInfo.name,
|
|
summary,
|
|
categories: [chapterInfo.category],
|
|
tags: ["pastry_chefs_little_black_book", chapterInfo.category, "classic"],
|
|
yield_quantity: Math.round(yieldQuantity * 100) / 100,
|
|
yield_unit_id: "gram",
|
|
yield_servings: yieldServings,
|
|
yield_basis: "theoretical",
|
|
yield: {
|
|
quantity: Math.round(yieldQuantity * 100) / 100,
|
|
unit_id: "gram",
|
|
servings: yieldServings,
|
|
basis: "theoretical",
|
|
},
|
|
components: formattedComponents,
|
|
steps: steps.length > 0 ? steps : [{ id: "step_1", order: 1, instruction: "Prepare formulation according to standard pastry method.", equipment_ids: [] }],
|
|
notes: allNotes,
|
|
shelf_life: parseShelfLife(chefNotesList),
|
|
});
|
|
}
|
|
}
|
|
|
|
return recipes;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Batch Ingestion Runner
|
|
// ---------------------------------------------------------------------------
|
|
async function main() {
|
|
const args = process.argv.slice(2);
|
|
const isSave = args.includes("--save");
|
|
const isDryRun = args.includes("--dry-run");
|
|
|
|
let targetPages = null;
|
|
const pageIdx = args.indexOf("--page");
|
|
if (pageIdx !== -1 && args[pageIdx + 1]) {
|
|
targetPages = [parseInt(args[pageIdx + 1], 10)];
|
|
}
|
|
const pagesIdx = args.indexOf("--pages");
|
|
if (pagesIdx !== -1 && args[pagesIdx + 1]) {
|
|
const [start, end] = args[pagesIdx + 1].split("-").map((n) => parseInt(n, 10));
|
|
targetPages = [];
|
|
for (let p = start; p <= end; p++) targetPages.push(p);
|
|
}
|
|
|
|
console.log("Loading Docling JSON from file.json...");
|
|
const rawData = fs.readFileSync(doclingPath, "utf8");
|
|
const doc = JSON.parse(rawData);
|
|
console.log(`Document loaded: ${doc.texts?.length || 0} texts, ${doc.tables?.length || 0} tables.`);
|
|
|
|
const recipes = parseDoclingBook(doc, targetPages);
|
|
console.log(`\nFound ${recipes.length} formulation(s).`);
|
|
|
|
// Detailed inspect for targeted page runs
|
|
if (targetPages && targetPages.length <= 5) {
|
|
for (const recipe of recipes) {
|
|
console.log(`\n================================================================`);
|
|
console.log(`📖 Page ${recipe.page_no}: ${recipe.title} (${recipe.chapter})`);
|
|
console.log(` ID: ${recipe.id}`);
|
|
console.log(` Categories: ${recipe.categories.join(", ")}`);
|
|
console.log(` Yield: ${recipe.yield.quantity} ${recipe.yield.unit_id} (${recipe.yield.basis})`);
|
|
if (recipe.summary) console.log(` Summary: ${recipe.summary}`);
|
|
if (recipe.shelf_life) console.log(` Shelf Life: ${recipe.shelf_life.quantity} ${recipe.shelf_life.unit} (${recipe.shelf_life.storage_condition})`);
|
|
|
|
for (const comp of recipe.components) {
|
|
console.log(`\n Component: [${comp.name}] (${comp.items.length} lines):`);
|
|
for (const item of comp.items) {
|
|
const pct = item.percentage !== null ? `(${item.percentage}%)` : "";
|
|
const basis = item.basis_member ? "[BASIS]" : "";
|
|
const note = item.notes ? `[${item.notes}]` : "";
|
|
console.log(` - ${item.name.padEnd(26)} ${String(item.quantity).padStart(5)} ${item.unit_id.padEnd(5)} ${pct.padStart(9)} ${basis} ${note}`);
|
|
}
|
|
}
|
|
|
|
console.log(`\n Procedure:`);
|
|
let stepNum = 1;
|
|
for (const step of recipe.steps) {
|
|
const isHeading = step.instruction.endsWith(":");
|
|
if (isHeading) {
|
|
console.log(`\n 📂 ${step.instruction}`);
|
|
stepNum = 1;
|
|
} else {
|
|
const eq = step.equipment_ids.length > 0 ? ` [Equip: ${step.equipment_ids.join(", ")}]` : "";
|
|
console.log(` ${stepNum++}. ${step.instruction}${eq}`);
|
|
}
|
|
}
|
|
|
|
if (recipe.notes.length > 0) {
|
|
console.log(`\n Chef's Notes:`);
|
|
for (const n of recipe.notes) console.log(` * ${n}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (isSave) {
|
|
console.log(`\n💾 Ingesting ${recipes.length} recipes into Formulation database...`);
|
|
const { createMcpTools } = await import("../src/mcp/tools.ts");
|
|
const { openDatabase, refreshSiteProjection } = await import("../src/lib/database.ts");
|
|
const db = openDatabase({ readOnly: false });
|
|
const tools = createMcpTools(() => db);
|
|
|
|
try {
|
|
let newIngredientsCount = 0;
|
|
const recipeIds = [];
|
|
|
|
for (let i = 0; i < recipes.length; i++) {
|
|
const recipe = recipes[i];
|
|
|
|
for (const comp of recipe.components) {
|
|
for (const item of comp.items) {
|
|
const exists = db.prepare("SELECT 1 FROM ingredients WHERE id = ?").get(item.ingredient_id);
|
|
if (!exists) {
|
|
tools.saveIngredient({
|
|
id: item.ingredient_id,
|
|
name: item.name,
|
|
categories: ["pantry", "baking", "imported_stub"],
|
|
});
|
|
newIngredientsCount++;
|
|
}
|
|
}
|
|
}
|
|
|
|
const res = tools.saveRecipe(recipe);
|
|
recipeIds.push(res.recipe_id);
|
|
|
|
if ((i + 1) % 50 === 0 || i + 1 === recipes.length) {
|
|
console.log(` [${i + 1}/${recipes.length}] Processed: ${recipe.title} (${res.recipe_id})`);
|
|
}
|
|
}
|
|
|
|
// Update or create Master Collection
|
|
if (!targetPages) {
|
|
const collectionId = "the_pastry_chefs_little_black_book_vol_1";
|
|
tools.saveRecipeBook({
|
|
id: collectionId,
|
|
name: "The Pastry Chef's Little Black Book (Vol. I)",
|
|
description: "Classic culinary pastry reference by Michael Zebrowski & Michael Mignano (477 formulations across 13 chapters).",
|
|
recipe_ids: recipeIds,
|
|
});
|
|
}
|
|
|
|
refreshSiteProjection(db);
|
|
|
|
console.log(`\n================================================================`);
|
|
console.log(`🎉 INGESTION COMPLETE!`);
|
|
console.log(`================================================================`);
|
|
console.log(` • Recipes Ingested: ${recipes.length}`);
|
|
console.log(` • New Ingredients Stubbed: ${newIngredientsCount}`);
|
|
console.log(` • Site Projection: Refreshed successfully`);
|
|
} catch (error) {
|
|
console.error("Ingestion failed:", error);
|
|
process.exit(1);
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error("Ingestion error:", err);
|
|
process.exit(1);
|
|
});
|