From 63492445a9e58fd6e33c16a7525c3911ab6aada5 Mon Sep 17 00:00:00 2001 From: Nicholas Ward Date: Mon, 17 Aug 2026 17:47:13 -0500 Subject: [PATCH] fix(ingest): support 2-page spreads and multi-component table structures in Docling parser --- scripts/ingest-docling-book.mjs | 295 +++++++++++++++++++++----------- 1 file changed, 194 insertions(+), 101 deletions(-) diff --git a/scripts/ingest-docling-book.mjs b/scripts/ingest-docling-book.mjs index 14a0599..801f781 100644 --- a/scripts/ingest-docling-book.mjs +++ b/scripts/ingest-docling-book.mjs @@ -1,9 +1,16 @@ #!/usr/bin/env node /** - * Batch Docling Recipe Ingestor for "The Pastry Chef's Little Black Book, Vol. I" + * Advanced Docling Recipe Ingestor for "The Pastry Chef's Little Black Book, Vol. I" + * + * Supports: + * - 2-page facing spreads (Table on Left, Procedure on Right) + * - Multi-component formulation splitting (Dough Packet, Butter Packet, Filling, Crust) + * - Fractional spoon & unit fallbacks across all columns + * - Multi-stage procedure preservation with equipment inference + * - Shelf-life & chef's notes extraction * * Usage: - * node scripts/ingest-docling-book.mjs --dry-run + * node scripts/ingest-docling-book.mjs --dry-run --pages 26-27 * node scripts/ingest-docling-book.mjs --save */ import fs from "node:fs"; @@ -100,12 +107,15 @@ const KNOWN_INGREDIENT_MAP = { "vanilla bean": "vanilla_bean", "vanilla beans": "vanilla_bean", "vanilla paste": "vanilla_extract", + "almond extract": "almond_extract", "cinnamon (ground)": "cinnamon", "cinnamon": "cinnamon", "ground cinnamon": "cinnamon", "nutmeg": "nutmeg", "ground nutmeg": "nutmeg", "black pepper": "black_pepper", + "white vinegar": "white_vinegar", + "vinegar": "white_vinegar", "water": "water", "water (cold)": "water", "water (warm)": "water", @@ -139,6 +149,7 @@ const KNOWN_INGREDIENT_MAP = { "olive oil": "olive_oil", "lemon juice": "lemon_juice", "lemon zest": "lemon_zest", + "lemons": "lemon", "orange juice": "orange_juice", "orange zest": "orange_zest", "lime juice": "lime_juice", @@ -211,13 +222,13 @@ function parseMetricAmount(str) { return null; } -function parseUsFallback(usText, name) { - if (!usText) { +function parseUsFallback(text, name) { + if (!text) { if (/zest/i.test(name)) return { quantity: 1, unit_id: "each", notes: "Zest of 1" }; if (/vanilla bean/i.test(name)) return { quantity: 1, unit_id: "each", notes: "1 bean" }; return { quantity: 1, unit_id: "gram", notes: "To taste / as needed" }; } - const s = usText.replace(/[\r\n\s]+/g, " ").trim().toLowerCase(); + const s = text.replace(/[\r\n\s]+/g, " ").trim().toLowerCase(); if (s.includes("1/8") || s.includes("⅛")) return { quantity: 0.6, unit_id: "gram", notes: "⅛ tsp" }; if (s.includes("1/4") || s.includes("¼")) return { quantity: 1.25, unit_id: "gram", notes: "¼ tsp" }; if (s.includes("1/2") || s.includes("½")) return { quantity: 2.5, unit_id: "gram", notes: "½ tsp" }; @@ -233,10 +244,10 @@ function parseUsFallback(usText, name) { const ozMatch = s.match(/^([\d.]+)\s*oz$/); if (ozMatch && parseFloat(ozMatch[1]) > 0) { - return { quantity: Math.round(parseFloat(ozMatch[1]) * 28.3495 * 100) / 100, unit_id: "gram", notes: usText }; + return { quantity: Math.round(parseFloat(ozMatch[1]) * 28.3495 * 100) / 100, unit_id: "gram", notes: text }; } - return { quantity: 1, unit_id: "gram", notes: usText }; + return { quantity: 1, unit_id: "gram", notes: text }; } function cleanIngredientName(raw) { @@ -361,6 +372,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) { const isIngredientTable = headerRow && [...headerRow.values()].some((v) => /ingredients/i.test(v)); if (!isIngredientTable) continue; + // Find Recipe Title on this page const titleNode = page.texts.find( (t) => t.label === "section_header" && !/procedure|chef's notes|notes|table of contents|scaling|baking/i.test(t.text) ); @@ -374,7 +386,11 @@ export function parseDoclingBook(doclingJson, targetPages = null) { usedSlugs.set(slugId, 1); } - const items = []; + // Parse ingredients into components + const components = []; + let currentComponent = { id: "main", name: "Main", items: [] }; + components.push(currentComponent); + let totalYieldGrams = null; for (let r = 1; r <= maxRow; r++) { @@ -386,24 +402,37 @@ export function parseDoclingBook(doclingJson, targetPages = null) { const usText = row.get(2) || ""; if (/total weight/i.test(ingText)) { - const parsedTotal = parseMetricAmount(metricText); + const parsedTotal = parseMetricAmount(metricText) || parseMetricAmount(usText); if (parsedTotal) totalYieldGrams = parsedTotal.quantity; continue; } if (!ingText) continue; - if (ingText.endsWith(":") && !metricText && !usText) continue; + + // Detect sub-component headers inside tables like "Dough Packet (Détrempe):" or "Filling:" + if (ingText.endsWith(":") && !metricText && !usText) { + const compName = ingText.replace(/:$/, "").trim(); + const compSlug = slugify(compName); + if (currentComponent.items.length === 0 && components.length === 1) { + currentComponent.id = compSlug; + currentComponent.name = compName; + } else { + currentComponent = { id: compSlug, name: compName, items: [] }; + components.push(currentComponent); + } + continue; + } const { name, notes: parenNotes } = cleanIngredientName(ingText); const ingredientId = inferIngredientId(name); - const parsedMetric = parseMetricAmount(metricText); + let parsedMetric = parseMetricAmount(metricText); let notes = parenNotes; let quantity = parsedMetric ? parsedMetric.quantity : 0; let unitId = parsedMetric ? parsedMetric.unit_id : "gram"; if (quantity <= 0) { - const fallback = parseUsFallback(usText, name); + const fallback = parseUsFallback(metricText || usText, name); quantity = fallback.quantity; unitId = fallback.unit_id; if (fallback.notes) { @@ -415,7 +444,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) { } } - items.push({ + currentComponent.items.push({ raw_name: ingText, clean_name: name, ingredient_id: ingredientId, @@ -427,40 +456,61 @@ export function parseDoclingBook(doclingJson, targetPages = null) { }); } - if (items.length === 0) continue; + // Filter out empty components + const validComponents = components.filter((c) => c.items.length > 0); + if (validComponents.length === 0) continue; - const flourBasisWeight = items + // Calculate Baker's Percentages across all components + const allItems = validComponents.flatMap((c) => c.items); + const flourBasisWeight = allItems .filter((i) => i.basis_member) .reduce((sum, i) => sum + i.quantity, 0); - const computedItems = items.map((item, idx) => { - let pct = null; - if (flourBasisWeight > 0 && item.quantity > 0) { - pct = Number(((item.quantity / flourBasisWeight) * 100).toFixed(2)); - } - return { - id: `line_${String(idx + 1).padStart(2, "0")}_${item.ingredient_id}`, - ingredient_id: item.ingredient_id, - name: titleCase(item.clean_name), - quantity: item.quantity, - unit_id: item.unit_id, - percentage: pct, - basis_member: item.basis_member, - notes: item.notes, - }; - }); + let itemCounter = 1; + const formattedComponents = validComponents.map((comp) => ({ + id: comp.id, + name: comp.name, + notes: [], + items: comp.items.map((item) => { + let pct = null; + if (flourBasisWeight > 0 && item.quantity > 0) { + pct = Number(((item.quantity / flourBasisWeight) * 100).toFixed(2)); + } + return { + id: `line_${String(itemCounter++).padStart(2, "0")}_${item.ingredient_id}`, + ingredient_id: item.ingredient_id, + name: titleCase(item.clean_name), + quantity: item.quantity, + unit_id: item.unit_id, + percentage: pct, + basis_member: item.basis_member, + notes: item.notes, + }; + }), + })); + + // Find procedure steps on current page or facing spread page (pageNo + 1) + const textSources = [...page.texts]; + const nextPage = pagesMap.get(pageNo + 1); + if (nextPage && nextPage.tables.length === 0) { + textSources.push(...nextPage.texts); + } const steps = []; + let currentSectionPrefix = ""; let inProcedure = false; let inChefNotes = false; const chefNotesList = []; const narrativeTexts = []; - for (const textNode of page.texts) { + for (const textNode of textSources) { const text = textNode.text.trim(); - if (/^procedure:?$/i.test(text)) { + + // Match procedure section headers like "Dough Packet Procedure:", "Assembly Procedure:", "Procedure:" + if (/procedure:?$/i.test(text)) { inProcedure = true; inChefNotes = false; + currentSectionPrefix = text.replace(/procedure:?$/i, "").trim(); continue; } if (/^chef's notes:?$/i.test(text)) { @@ -471,12 +521,15 @@ export function parseDoclingBook(doclingJson, targetPages = null) { if (inProcedure) { if (textNode.label === "list_item" || textNode.label === "text") { - steps.push({ - id: `step_${steps.length + 1}`, - order: steps.length + 1, - instruction: text, - equipment_ids: inferEquipment(text), - }); + if (!/^\d+$/.test(text)) { + const prefix = currentSectionPrefix ? `[${currentSectionPrefix}] ` : ""; + steps.push({ + id: `step_${steps.length + 1}`, + order: steps.length + 1, + instruction: `${prefix}${text}`, + equipment_ids: inferEquipment(text), + }); + } } } else if (inChefNotes) { if (textNode.label === "list_item") { @@ -487,7 +540,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) { } } - const sumWeight = computedItems.reduce((s, i) => s + (i.unit_id === "gram" ? i.quantity : 0), 0); + const sumWeight = allItems.reduce((s, i) => s + (i.unit_id === "gram" ? i.quantity : 0), 0); const yieldQuantity = totalYieldGrams || (sumWeight > 0 ? sumWeight : 1000); recipes.push({ @@ -507,13 +560,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) { unit_id: "gram", basis: "theoretical", }, - components: [ - { - id: "main", - name: "Main", - items: computedItems, - }, - ], + components: formattedComponents, steps: steps.length > 0 ? steps : [{ id: "step_1", order: 1, instruction: "Prepare formulation according to standard pastry method.", equipment_ids: [] }], notes: chefNotesList, shelf_life: parseShelfLife(chefNotesList), @@ -530,77 +577,123 @@ export function parseDoclingBook(doclingJson, targetPages = null) { async function main() { const args = process.argv.slice(2); const isSave = args.includes("--save"); + const isDryRun = args.includes("--dry-run"); + + let targetPages = null; + const pageIdx = args.indexOf("--page"); + if (pageIdx !== -1 && args[pageIdx + 1]) { + targetPages = [parseInt(args[pageIdx + 1], 10)]; + } + const pagesIdx = args.indexOf("--pages"); + if (pagesIdx !== -1 && args[pagesIdx + 1]) { + const [start, end] = args[pagesIdx + 1].split("-").map((n) => parseInt(n, 10)); + targetPages = []; + for (let p = start; p <= end; p++) targetPages.push(p); + } console.log("Loading Docling JSON from file.json..."); const rawData = fs.readFileSync(doclingPath, "utf8"); const doc = JSON.parse(rawData); console.log(`Document loaded: ${doc.texts?.length || 0} texts, ${doc.tables?.length || 0} tables.`); - const recipes = parseDoclingBook(doc); - console.log(`\nFound ${recipes.length} formulation(s) across 13 chapters.`); + const recipes = parseDoclingBook(doc, targetPages); + console.log(`\nFound ${recipes.length} formulation(s).`); - if (!isSave) { - console.log("Run with '--save' to commit to database."); - return; - } + // Detailed inspect for targeted page runs + if (targetPages && targetPages.length <= 5) { + for (const recipe of recipes) { + console.log(`\n================================================================`); + console.log(`📖 Page ${recipe.page_no}: ${recipe.title} (${recipe.chapter})`); + console.log(` ID: ${recipe.id}`); + console.log(` Categories: ${recipe.categories.join(", ")}`); + console.log(` Yield: ${recipe.yield.quantity} ${recipe.yield.unit_id} (${recipe.yield.basis})`); + if (recipe.summary) console.log(` Summary: ${recipe.summary}`); + if (recipe.shelf_life) console.log(` Shelf Life: ${recipe.shelf_life.quantity} ${recipe.shelf_life.unit} (${recipe.shelf_life.storage_condition})`); - console.log(`\n💾 Ingesting ${recipes.length} recipes into Formulation database...`); - const { createMcpTools } = await import("../src/mcp/tools.ts"); - const { openDatabase, refreshSiteProjection } = await import("../src/lib/database.ts"); - const db = openDatabase({ readOnly: false }); - const tools = createMcpTools(() => db); - - try { - let newIngredientsCount = 0; - const recipeIds = []; - - for (let i = 0; i < recipes.length; i++) { - const recipe = recipes[i]; - - // Auto-provision lightweight ingredient records - for (const item of recipe.components[0].items) { - const exists = db.prepare("SELECT 1 FROM ingredients WHERE id = ?").get(item.ingredient_id); - if (!exists) { - tools.saveIngredient({ - id: item.ingredient_id, - name: item.name, - categories: ["pantry", "baking", "imported_stub"], - }); - newIngredientsCount++; + for (const comp of recipe.components) { + console.log(`\n Component: [${comp.name}] (${comp.items.length} lines):`); + for (const item of comp.items) { + const pct = item.percentage !== null ? `(${item.percentage}%)` : ""; + const basis = item.basis_member ? "[BASIS]" : ""; + const note = item.notes ? `[${item.notes}]` : ""; + console.log(` - ${item.name.padEnd(26)} ${String(item.quantity).padStart(5)} ${item.unit_id.padEnd(5)} ${pct.padStart(9)} ${basis} ${note}`); } } - const res = tools.saveRecipe(recipe); - recipeIds.push(res.recipe_id); + console.log(`\n Procedure (${recipe.steps.length} steps):`); + for (const step of recipe.steps) { + const eq = step.equipment_ids.length > 0 ? ` [Equip: ${step.equipment_ids.join(", ")}]` : ""; + console.log(` ${step.order}. ${step.instruction}${eq}`); + } - if ((i + 1) % 50 === 0 || i + 1 === recipes.length) { - console.log(` [${i + 1}/${recipes.length}] Processed: ${recipe.title} (${res.recipe_id})`); + if (recipe.notes.length > 0) { + console.log(`\n Chef's Notes:`); + for (const n of recipe.notes) console.log(` * ${n}`); } } + } - // Create or update collection: "The Pastry Chef's Little Black Book (Vol. I)" - const collectionId = "the_pastry_chefs_little_black_book_vol_1"; - tools.saveRecipeBook({ - id: collectionId, - name: "The Pastry Chef's Little Black Book (Vol. I)", - description: "Classic culinary pastry reference by Michael Zebrowski & Michael Mignano (477 formulations across 13 chapters).", - recipe_ids: recipeIds, - }); + if (isSave) { + console.log(`\n💾 Ingesting ${recipes.length} recipes into Formulation database...`); + const { createMcpTools } = await import("../src/mcp/tools.ts"); + const { openDatabase, refreshSiteProjection } = await import("../src/lib/database.ts"); + const db = openDatabase({ readOnly: false }); + const tools = createMcpTools(() => db); - refreshSiteProjection(db); + try { + let newIngredientsCount = 0; + const recipeIds = []; - console.log(`\n================================================================`); - console.log(`🎉 INGESTION COMPLETE!`); - console.log(`================================================================`); - console.log(` • Recipes Ingested: ${recipes.length}`); - console.log(` • New Ingredients Stubbed: ${newIngredientsCount}`); - console.log(` • Collection Created: The Pastry Chef's Little Black Book (Vol. I) (${collectionId})`); - console.log(` • Site Projection: Refreshed successfully`); - } catch (error) { - console.error("Ingestion failed:", error); - process.exit(1); - } finally { - db.close(); + for (let i = 0; i < recipes.length; i++) { + const recipe = recipes[i]; + + for (const comp of recipe.components) { + for (const item of comp.items) { + const exists = db.prepare("SELECT 1 FROM ingredients WHERE id = ?").get(item.ingredient_id); + if (!exists) { + tools.saveIngredient({ + id: item.ingredient_id, + name: item.name, + categories: ["pantry", "baking", "imported_stub"], + }); + newIngredientsCount++; + } + } + } + + const res = tools.saveRecipe(recipe); + recipeIds.push(res.recipe_id); + + if ((i + 1) % 50 === 0 || i + 1 === recipes.length) { + console.log(` [${i + 1}/${recipes.length}] Processed: ${recipe.title} (${res.recipe_id})`); + } + } + + // Update or create Master Collection + if (!targetPages) { + const collectionId = "the_pastry_chefs_little_black_book_vol_1"; + tools.saveRecipeBook({ + id: collectionId, + name: "The Pastry Chef's Little Black Book (Vol. I)", + description: "Classic culinary pastry reference by Michael Zebrowski & Michael Mignano (477 formulations across 13 chapters).", + recipe_ids: recipeIds, + }); + } + + refreshSiteProjection(db); + + console.log(`\n================================================================`); + console.log(`🎉 INGESTION COMPLETE!`); + console.log(`================================================================`); + console.log(` • Recipes Ingested: ${recipes.length}`); + console.log(` • New Ingredients Stubbed: ${newIngredientsCount}`); + console.log(` • Site Projection: Refreshed successfully`); + } catch (error) { + console.error("Ingestion failed:", error); + process.exit(1); + } finally { + db.close(); + } } }