fix(ingest): support 2-page spreads and multi-component table structures in Docling parser

scripts/ingest-docling-book.mjs

@@ -1,9 +1,16 @@
 #!/usr/bin/env node
 /**
- * Batch Docling Recipe Ingestor for "The Pastry Chef's Little Black Book, Vol. I"
+ * Advanced Docling Recipe Ingestor for "The Pastry Chef's Little Black Book, Vol. I"
+ *
+ * Supports:
+ * - 2-page facing spreads (Table on Left, Procedure on Right)
+ * - Multi-component formulation splitting (Dough Packet, Butter Packet, Filling, Crust)
+ * - Fractional spoon & unit fallbacks across all columns
+ * - Multi-stage procedure preservation with equipment inference
+ * - Shelf-life & chef's notes extraction
  *
  * Usage:
- *   node scripts/ingest-docling-book.mjs --dry-run
+ *   node scripts/ingest-docling-book.mjs --dry-run --pages 26-27
  *   node scripts/ingest-docling-book.mjs --save
  */
 import fs from "node:fs";
@@ -100,12 +107,15 @@ const KNOWN_INGREDIENT_MAP = {
   "vanilla bean": "vanilla_bean",
   "vanilla beans": "vanilla_bean",
   "vanilla paste": "vanilla_extract",
+  "almond extract": "almond_extract",
   "cinnamon (ground)": "cinnamon",
   "cinnamon": "cinnamon",
   "ground cinnamon": "cinnamon",
   "nutmeg": "nutmeg",
   "ground nutmeg": "nutmeg",
   "black pepper": "black_pepper",
+  "white vinegar": "white_vinegar",
+  "vinegar": "white_vinegar",
   "water": "water",
   "water (cold)": "water",
   "water (warm)": "water",
@@ -139,6 +149,7 @@ const KNOWN_INGREDIENT_MAP = {
   "olive oil": "olive_oil",
   "lemon juice": "lemon_juice",
   "lemon zest": "lemon_zest",
+  "lemons": "lemon",
   "orange juice": "orange_juice",
   "orange zest": "orange_zest",
   "lime juice": "lime_juice",
@@ -211,13 +222,13 @@ function parseMetricAmount(str) {
   return null;
 }
 
-function parseUsFallback(usText, name) {
+function parseUsFallback(text, name) {
-  if (!usText) {
+  if (!text) {
     if (/zest/i.test(name)) return { quantity: 1, unit_id: "each", notes: "Zest of 1" };
     if (/vanilla bean/i.test(name)) return { quantity: 1, unit_id: "each", notes: "1 bean" };
     return { quantity: 1, unit_id: "gram", notes: "To taste / as needed" };
   }
-  const s = usText.replace(/[\r\n\s]+/g, " ").trim().toLowerCase();
+  const s = text.replace(/[\r\n\s]+/g, " ").trim().toLowerCase();
   if (s.includes("1/8") || s.includes("⅛")) return { quantity: 0.6, unit_id: "gram", notes: "⅛ tsp" };
   if (s.includes("1/4") || s.includes("¼")) return { quantity: 1.25, unit_id: "gram", notes: "¼ tsp" };
   if (s.includes("1/2") || s.includes("½")) return { quantity: 2.5, unit_id: "gram", notes: "½ tsp" };
@@ -233,10 +244,10 @@ function parseUsFallback(usText, name) {
 
   const ozMatch = s.match(/^([\d.]+)\s*oz$/);
   if (ozMatch && parseFloat(ozMatch[1]) > 0) {
-    return { quantity: Math.round(parseFloat(ozMatch[1]) * 28.3495 * 100) / 100, unit_id: "gram", notes: usText };
+    return { quantity: Math.round(parseFloat(ozMatch[1]) * 28.3495 * 100) / 100, unit_id: "gram", notes: text };
   }
 
-  return { quantity: 1, unit_id: "gram", notes: usText };
+  return { quantity: 1, unit_id: "gram", notes: text };
 }
 
 function cleanIngredientName(raw) {
@@ -361,6 +372,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
       const isIngredientTable = headerRow && [...headerRow.values()].some((v) => /ingredients/i.test(v));
       if (!isIngredientTable) continue;
 
+      // Find Recipe Title on this page
       const titleNode = page.texts.find(
         (t) => t.label === "section_header" && !/procedure|chef's notes|notes|table of contents|scaling|baking/i.test(t.text)
       );
@@ -374,7 +386,11 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
         usedSlugs.set(slugId, 1);
       }
 
-      const items = [];
+      // Parse ingredients into components
+      const components = [];
+      let currentComponent = { id: "main", name: "Main", items: [] };
+      components.push(currentComponent);
+
       let totalYieldGrams = null;
 
       for (let r = 1; r <= maxRow; r++) {
@@ -386,24 +402,37 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
         const usText = row.get(2) || "";
 
         if (/total weight/i.test(ingText)) {
-          const parsedTotal = parseMetricAmount(metricText);
+          const parsedTotal = parseMetricAmount(metricText) || parseMetricAmount(usText);
           if (parsedTotal) totalYieldGrams = parsedTotal.quantity;
           continue;
         }
 
         if (!ingText) continue;
-        if (ingText.endsWith(":") && !metricText && !usText) continue;
+
+        // Detect sub-component headers inside tables like "Dough Packet (Détrempe):" or "Filling:"
+        if (ingText.endsWith(":") && !metricText && !usText) {
+          const compName = ingText.replace(/:$/, "").trim();
+          const compSlug = slugify(compName);
+          if (currentComponent.items.length === 0 && components.length === 1) {
+            currentComponent.id = compSlug;
+            currentComponent.name = compName;
+          } else {
+            currentComponent = { id: compSlug, name: compName, items: [] };
+            components.push(currentComponent);
+          }
+          continue;
+        }
 
         const { name, notes: parenNotes } = cleanIngredientName(ingText);
         const ingredientId = inferIngredientId(name);
-        const parsedMetric = parseMetricAmount(metricText);
 
+        let parsedMetric = parseMetricAmount(metricText);
         let notes = parenNotes;
         let quantity = parsedMetric ? parsedMetric.quantity : 0;
         let unitId = parsedMetric ? parsedMetric.unit_id : "gram";
 
         if (quantity <= 0) {
-          const fallback = parseUsFallback(usText, name);
+          const fallback = parseUsFallback(metricText || usText, name);
           quantity = fallback.quantity;
           unitId = fallback.unit_id;
           if (fallback.notes) {
@@ -415,7 +444,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
           }
         }
 
-        items.push({
+        currentComponent.items.push({
           raw_name: ingText,
           clean_name: name,
           ingredient_id: ingredientId,
@@ -427,19 +456,28 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
         });
       }
 
-      if (items.length === 0) continue;
+      // Filter out empty components
+      const validComponents = components.filter((c) => c.items.length > 0);
+      if (validComponents.length === 0) continue;
 
-      const flourBasisWeight = items
+      // Calculate Baker's Percentages across all components
+      const allItems = validComponents.flatMap((c) => c.items);
+      const flourBasisWeight = allItems
         .filter((i) => i.basis_member)
         .reduce((sum, i) => sum + i.quantity, 0);
 
-      const computedItems = items.map((item, idx) => {
+      let itemCounter = 1;
+      const formattedComponents = validComponents.map((comp) => ({
+        id: comp.id,
+        name: comp.name,
+        notes: [],
+        items: comp.items.map((item) => {
           let pct = null;
           if (flourBasisWeight > 0 && item.quantity > 0) {
             pct = Number(((item.quantity / flourBasisWeight) * 100).toFixed(2));
           }
           return {
-          id: `line_${String(idx + 1).padStart(2, "0")}_${item.ingredient_id}`,
+            id: `line_${String(itemCounter++).padStart(2, "0")}_${item.ingredient_id}`,
             ingredient_id: item.ingredient_id,
             name: titleCase(item.clean_name),
             quantity: item.quantity,
@@ -448,19 +486,31 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
             basis_member: item.basis_member,
             notes: item.notes,
           };
-      });
+        }),
+      }));
+
+      // Find procedure steps on current page or facing spread page (pageNo + 1)
+      const textSources = [...page.texts];
+      const nextPage = pagesMap.get(pageNo + 1);
+      if (nextPage && nextPage.tables.length === 0) {
+        textSources.push(...nextPage.texts);
+      }
 
       const steps = [];
+      let currentSectionPrefix = "";
       let inProcedure = false;
       let inChefNotes = false;
       const chefNotesList = [];
       const narrativeTexts = [];
 
-      for (const textNode of page.texts) {
+      for (const textNode of textSources) {
         const text = textNode.text.trim();
-        if (/^procedure:?$/i.test(text)) {
+        
+        // Match procedure section headers like "Dough Packet Procedure:", "Assembly Procedure:", "Procedure:"
+        if (/procedure:?$/i.test(text)) {
           inProcedure = true;
           inChefNotes = false;
+          currentSectionPrefix = text.replace(/procedure:?$/i, "").trim();
           continue;
         }
         if (/^chef's notes:?$/i.test(text)) {
@@ -471,13 +521,16 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
 
         if (inProcedure) {
           if (textNode.label === "list_item" || textNode.label === "text") {
+            if (!/^\d+$/.test(text)) {
+              const prefix = currentSectionPrefix ? `[${currentSectionPrefix}] ` : "";
               steps.push({
                 id: `step_${steps.length + 1}`,
                 order: steps.length + 1,
-              instruction: text,
+                instruction: `${prefix}${text}`,
                 equipment_ids: inferEquipment(text),
               });
             }
+          }
         } else if (inChefNotes) {
           if (textNode.label === "list_item") {
             chefNotesList.push(text);
@@ -487,7 +540,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
         }
       }
 
-      const sumWeight = computedItems.reduce((s, i) => s + (i.unit_id === "gram" ? i.quantity : 0), 0);
+      const sumWeight = allItems.reduce((s, i) => s + (i.unit_id === "gram" ? i.quantity : 0), 0);
       const yieldQuantity = totalYieldGrams || (sumWeight > 0 ? sumWeight : 1000);
 
       recipes.push({
@@ -507,13 +560,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
           unit_id: "gram",
           basis: "theoretical",
         },
-        components: [
+        components: formattedComponents,
-          {
-            id: "main",
-            name: "Main",
-            items: computedItems,
-          },
-        ],
         steps: steps.length > 0 ? steps : [{ id: "step_1", order: 1, instruction: "Prepare formulation according to standard pastry method.", equipment_ids: [] }],
         notes: chefNotesList,
         shelf_life: parseShelfLife(chefNotesList),
@@ -530,20 +577,63 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
 async function main() {
   const args = process.argv.slice(2);
   const isSave = args.includes("--save");
+  const isDryRun = args.includes("--dry-run");
+
+  let targetPages = null;
+  const pageIdx = args.indexOf("--page");
+  if (pageIdx !== -1 && args[pageIdx + 1]) {
+    targetPages = [parseInt(args[pageIdx + 1], 10)];
+  }
+  const pagesIdx = args.indexOf("--pages");
+  if (pagesIdx !== -1 && args[pagesIdx + 1]) {
+    const [start, end] = args[pagesIdx + 1].split("-").map((n) => parseInt(n, 10));
+    targetPages = [];
+    for (let p = start; p <= end; p++) targetPages.push(p);
+  }
 
   console.log("Loading Docling JSON from file.json...");
   const rawData = fs.readFileSync(doclingPath, "utf8");
   const doc = JSON.parse(rawData);
   console.log(`Document loaded: ${doc.texts?.length || 0} texts, ${doc.tables?.length || 0} tables.`);
 
-  const recipes = parseDoclingBook(doc);
+  const recipes = parseDoclingBook(doc, targetPages);
-  console.log(`\nFound ${recipes.length} formulation(s) across 13 chapters.`);
+  console.log(`\nFound ${recipes.length} formulation(s).`);
 
-  if (!isSave) {
+  // Detailed inspect for targeted page runs
-    console.log("Run with '--save' to commit to database.");
+  if (targetPages && targetPages.length <= 5) {
-    return;
+    for (const recipe of recipes) {
+      console.log(`\n================================================================`);
+      console.log(`📖 Page ${recipe.page_no}: ${recipe.title} (${recipe.chapter})`);
+      console.log(`   ID:         ${recipe.id}`);
+      console.log(`   Categories: ${recipe.categories.join(", ")}`);
+      console.log(`   Yield:      ${recipe.yield.quantity} ${recipe.yield.unit_id} (${recipe.yield.basis})`);
+      if (recipe.summary) console.log(`   Summary:    ${recipe.summary}`);
+      if (recipe.shelf_life) console.log(`   Shelf Life: ${recipe.shelf_life.quantity} ${recipe.shelf_life.unit} (${recipe.shelf_life.storage_condition})`);
+
+      for (const comp of recipe.components) {
+        console.log(`\n   Component: [${comp.name}] (${comp.items.length} lines):`);
+        for (const item of comp.items) {
+          const pct = item.percentage !== null ? `(${item.percentage}%)` : "";
+          const basis = item.basis_member ? "[BASIS]" : "";
+          const note = item.notes ? `[${item.notes}]` : "";
+          console.log(`     - ${item.name.padEnd(26)} ${String(item.quantity).padStart(5)} ${item.unit_id.padEnd(5)} ${pct.padStart(9)} ${basis} ${note}`);
+        }
       }
 
+      console.log(`\n   Procedure (${recipe.steps.length} steps):`);
+      for (const step of recipe.steps) {
+        const eq = step.equipment_ids.length > 0 ? ` [Equip: ${step.equipment_ids.join(", ")}]` : "";
+        console.log(`     ${step.order}. ${step.instruction}${eq}`);
+      }
+
+      if (recipe.notes.length > 0) {
+        console.log(`\n   Chef's Notes:`);
+        for (const n of recipe.notes) console.log(`     * ${n}`);
+      }
+    }
+  }
+
+  if (isSave) {
     console.log(`\n💾 Ingesting ${recipes.length} recipes into Formulation database...`);
     const { createMcpTools } = await import("../src/mcp/tools.ts");
     const { openDatabase, refreshSiteProjection } = await import("../src/lib/database.ts");
@@ -557,8 +647,8 @@ async function main() {
       for (let i = 0; i < recipes.length; i++) {
         const recipe = recipes[i];
 
-      // Auto-provision lightweight ingredient records
+        for (const comp of recipe.components) {
-      for (const item of recipe.components[0].items) {
+          for (const item of comp.items) {
             const exists = db.prepare("SELECT 1 FROM ingredients WHERE id = ?").get(item.ingredient_id);
             if (!exists) {
               tools.saveIngredient({
@@ -569,6 +659,7 @@ async function main() {
               newIngredientsCount++;
             }
           }
+        }
 
         const res = tools.saveRecipe(recipe);
         recipeIds.push(res.recipe_id);
@@ -578,7 +669,8 @@ async function main() {
         }
       }
 
-    // Create or update collection: "The Pastry Chef's Little Black Book (Vol. I)"
+      // Update or create Master Collection
+      if (!targetPages) {
         const collectionId = "the_pastry_chefs_little_black_book_vol_1";
         tools.saveRecipeBook({
           id: collectionId,
@@ -586,6 +678,7 @@ async function main() {
           description: "Classic culinary pastry reference by Michael Zebrowski & Michael Mignano (477 formulations across 13 chapters).",
           recipe_ids: recipeIds,
         });
+      }
 
       refreshSiteProjection(db);
 
@@ -594,7 +687,6 @@ async function main() {
       console.log(`================================================================`);
       console.log(`  • Recipes Ingested:          ${recipes.length}`);
       console.log(`  • New Ingredients Stubbed:   ${newIngredientsCount}`);
-    console.log(`  • Collection Created:        The Pastry Chef's Little Black Book (Vol. I) (${collectionId})`);
       console.log(`  • Site Projection:           Refreshed successfully`);
     } catch (error) {
       console.error("Ingestion failed:", error);
@@ -603,6 +695,7 @@ async function main() {
       db.close();
     }
   }
+}
 
 main().catch((err) => {
   console.error("Ingestion error:", err);