nicholas/formulation
1
0
Fork 0
Code Pull Requests Actions Activity

fix(ingest): support 2-page spreads and multi-component table structures in Docling parser

Browse Source
This commit is contained in:
nicholas
2026-08-17 17:47:13 -05:00
parent 68d5b2a45f
commit 63492445a9
1 changed files with 194 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ingest-docling-book.mjs
+131 -38
View File
@@ -1,9 +1,16 @@
#!/usr/bin/env node
/**
* Batch Docling Recipe Ingestor for "The Pastry Chef's Little Black Book, Vol. I"
* Advanced Docling Recipe Ingestor for "The Pastry Chef's Little Black Book, Vol. I"
*
* Supports:
* - 2-page facing spreads (Table on Left, Procedure on Right)
* - Multi-component formulation splitting (Dough Packet, Butter Packet, Filling, Crust)
* - Fractional spoon & unit fallbacks across all columns
* - Multi-stage procedure preservation with equipment inference
* - Shelf-life & chef's notes extraction
*
* Usage:
* node scripts/ingest-docling-book.mjs --dry-run
* node scripts/ingest-docling-book.mjs --dry-run --pages 26-27
* node scripts/ingest-docling-book.mjs --save
*/
import fs from "node:fs";
@@ -100,12 +107,15 @@ const KNOWN_INGREDIENT_MAP = {
"vanilla bean": "vanilla_bean",
"vanilla beans": "vanilla_bean",
"vanilla paste": "vanilla_extract",
"almond extract": "almond_extract",
"cinnamon (ground)": "cinnamon",
"cinnamon": "cinnamon",
"ground cinnamon": "cinnamon",
"nutmeg": "nutmeg",
"ground nutmeg": "nutmeg",
"black pepper": "black_pepper",
"white vinegar": "white_vinegar",
"vinegar": "white_vinegar",
"water": "water",
"water (cold)": "water",
"water (warm)": "water",
@@ -139,6 +149,7 @@ const KNOWN_INGREDIENT_MAP = {
"olive oil": "olive_oil",
"lemon juice": "lemon_juice",
"lemon zest": "lemon_zest",
"lemons": "lemon",
"orange juice": "orange_juice",
"orange zest": "orange_zest",
"lime juice": "lime_juice",
@@ -211,13 +222,13 @@ function parseMetricAmount(str) {
return null;
}
function parseUsFallback(usText, name) {
if (!usText) {
function parseUsFallback(text, name) {
if (!text) {
if (/zest/i.test(name)) return { quantity: 1, unit_id: "each", notes: "Zest of 1" };
if (/vanilla bean/i.test(name)) return { quantity: 1, unit_id: "each", notes: "1 bean" };
return { quantity: 1, unit_id: "gram", notes: "To taste / as needed" };
}
const s = usText.replace(/[\r\n\s]+/g, " ").trim().toLowerCase();
const s = text.replace(/[\r\n\s]+/g, " ").trim().toLowerCase();
if (s.includes("1/8") || s.includes("⅛")) return { quantity: 0.6, unit_id: "gram", notes: "⅛ tsp" };
if (s.includes("1/4") || s.includes("¼")) return { quantity: 1.25, unit_id: "gram", notes: "¼ tsp" };
if (s.includes("1/2") || s.includes("½")) return { quantity: 2.5, unit_id: "gram", notes: "½ tsp" };
@@ -233,10 +244,10 @@ function parseUsFallback(usText, name) {
const ozMatch = s.match(/^([\d.]+)\s*oz$/);
if (ozMatch && parseFloat(ozMatch[1]) > 0) {
return { quantity: Math.round(parseFloat(ozMatch[1]) * 28.3495 * 100) / 100, unit_id: "gram", notes: usText };
return { quantity: Math.round(parseFloat(ozMatch[1]) * 28.3495 * 100) / 100, unit_id: "gram", notes: text };
}
return { quantity: 1, unit_id: "gram", notes: usText };
return { quantity: 1, unit_id: "gram", notes: text };
}
function cleanIngredientName(raw) {
@@ -361,6 +372,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
const isIngredientTable = headerRow && [...headerRow.values()].some((v) => /ingredients/i.test(v));
if (!isIngredientTable) continue;
// Find Recipe Title on this page
const titleNode = page.texts.find(
(t) => t.label === "section_header" && !/procedure|chef's notes|notes|table of contents|scaling|baking/i.test(t.text)
);
@@ -374,7 +386,11 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
usedSlugs.set(slugId, 1);
}
const items = [];
// Parse ingredients into components
const components = [];
let currentComponent = { id: "main", name: "Main", items: [] };
components.push(currentComponent);
let totalYieldGrams = null;
for (let r = 1; r <= maxRow; r++) {
@@ -386,24 +402,37 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
const usText = row.get(2) || "";
if (/total weight/i.test(ingText)) {
const parsedTotal = parseMetricAmount(metricText);
const parsedTotal = parseMetricAmount(metricText) || parseMetricAmount(usText);
if (parsedTotal) totalYieldGrams = parsedTotal.quantity;
continue;
}
if (!ingText) continue;
if (ingText.endsWith(":") && !metricText && !usText) continue;
// Detect sub-component headers inside tables like "Dough Packet (Détrempe):" or "Filling:"
if (ingText.endsWith(":") && !metricText && !usText) {
const compName = ingText.replace(/:$/, "").trim();
const compSlug = slugify(compName);
if (currentComponent.items.length === 0 && components.length === 1) {
currentComponent.id = compSlug;
currentComponent.name = compName;
} else {
currentComponent = { id: compSlug, name: compName, items: [] };
components.push(currentComponent);
}
continue;
}
const { name, notes: parenNotes } = cleanIngredientName(ingText);
const ingredientId = inferIngredientId(name);
const parsedMetric = parseMetricAmount(metricText);
let parsedMetric = parseMetricAmount(metricText);
let notes = parenNotes;
let quantity = parsedMetric ? parsedMetric.quantity : 0;
let unitId = parsedMetric ? parsedMetric.unit_id : "gram";
if (quantity <= 0) {
const fallback = parseUsFallback(usText, name);
const fallback = parseUsFallback(metricText || usText, name);
quantity = fallback.quantity;
unitId = fallback.unit_id;
if (fallback.notes) {
@@ -415,7 +444,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
}
}
items.push({
currentComponent.items.push({
raw_name: ingText,
clean_name: name,
ingredient_id: ingredientId,
@@ -427,19 +456,28 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
});
}
if (items.length === 0) continue;
// Filter out empty components
const validComponents = components.filter((c) => c.items.length > 0);
if (validComponents.length === 0) continue;
const flourBasisWeight = items
// Calculate Baker's Percentages across all components
const allItems = validComponents.flatMap((c) => c.items);
const flourBasisWeight = allItems
.filter((i) => i.basis_member)
.reduce((sum, i) => sum + i.quantity, 0);
const computedItems = items.map((item, idx) => {
let itemCounter = 1;
const formattedComponents = validComponents.map((comp) => ({
id: comp.id,
name: comp.name,
notes: [],
items: comp.items.map((item) => {
let pct = null;
if (flourBasisWeight > 0 && item.quantity > 0) {
pct = Number(((item.quantity / flourBasisWeight) * 100).toFixed(2));
}
return {
id: `line_${String(idx + 1).padStart(2, "0")}_${item.ingredient_id}`,
id: `line_${String(itemCounter++).padStart(2, "0")}_${item.ingredient_id}`,
ingredient_id: item.ingredient_id,
name: titleCase(item.clean_name),
quantity: item.quantity,
@@ -448,19 +486,31 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
basis_member: item.basis_member,
notes: item.notes,
};
});
}),
}));
// Find procedure steps on current page or facing spread page (pageNo + 1)
const textSources = [...page.texts];
const nextPage = pagesMap.get(pageNo + 1);
if (nextPage && nextPage.tables.length === 0) {
textSources.push(...nextPage.texts);
}
const steps = [];
let currentSectionPrefix = "";
let inProcedure = false;
let inChefNotes = false;
const chefNotesList = [];
const narrativeTexts = [];
for (const textNode of page.texts) {
for (const textNode of textSources) {
const text = textNode.text.trim();
if (/^procedure:?$/i.test(text)) {
// Match procedure section headers like "Dough Packet Procedure:", "Assembly Procedure:", "Procedure:"
if (/procedure:?$/i.test(text)) {
inProcedure = true;
inChefNotes = false;
currentSectionPrefix = text.replace(/procedure:?$/i, "").trim();
continue;
}
if (/^chef's notes:?$/i.test(text)) {
@@ -471,13 +521,16 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
if (inProcedure) {
if (textNode.label === "list_item" || textNode.label === "text") {
if (!/^\d+$/.test(text)) {
const prefix = currentSectionPrefix ? `[${currentSectionPrefix}] ` : "";
steps.push({
id: `step_${steps.length + 1}`,
order: steps.length + 1,
instruction: text,
instruction: `${prefix}${text}`,
equipment_ids: inferEquipment(text),
});
}
}
} else if (inChefNotes) {
if (textNode.label === "list_item") {
chefNotesList.push(text);
@@ -487,7 +540,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
}
}
const sumWeight = computedItems.reduce((s, i) => s + (i.unit_id === "gram" ? i.quantity : 0), 0);
const sumWeight = allItems.reduce((s, i) => s + (i.unit_id === "gram" ? i.quantity : 0), 0);
const yieldQuantity = totalYieldGrams || (sumWeight > 0 ? sumWeight : 1000);
recipes.push({
@@ -507,13 +560,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
unit_id: "gram",
basis: "theoretical",
},
components: [
{
id: "main",
name: "Main",
items: computedItems,
},
],
components: formattedComponents,
steps: steps.length > 0 ? steps : [{ id: "step_1", order: 1, instruction: "Prepare formulation according to standard pastry method.", equipment_ids: [] }],
notes: chefNotesList,
shelf_life: parseShelfLife(chefNotesList),
@@ -530,20 +577,63 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
async function main() {
const args = process.argv.slice(2);
const isSave = args.includes("--save");
const isDryRun = args.includes("--dry-run");
let targetPages = null;
const pageIdx = args.indexOf("--page");
if (pageIdx !== -1 && args[pageIdx + 1]) {
targetPages = [parseInt(args[pageIdx + 1], 10)];
}
const pagesIdx = args.indexOf("--pages");
if (pagesIdx !== -1 && args[pagesIdx + 1]) {
const [start, end] = args[pagesIdx + 1].split("-").map((n) => parseInt(n, 10));
targetPages = [];
for (let p = start; p <= end; p++) targetPages.push(p);
}
console.log("Loading Docling JSON from file.json...");
const rawData = fs.readFileSync(doclingPath, "utf8");
const doc = JSON.parse(rawData);
console.log(`Document loaded: ${doc.texts?.length || 0} texts, ${doc.tables?.length || 0} tables.`);
const recipes = parseDoclingBook(doc);
console.log(`\nFound ${recipes.length} formulation(s) across 13 chapters.`);
const recipes = parseDoclingBook(doc, targetPages);
console.log(`\nFound ${recipes.length} formulation(s).`);
if (!isSave) {
console.log("Run with '--save' to commit to database.");
return;
// Detailed inspect for targeted page runs
if (targetPages && targetPages.length <= 5) {
for (const recipe of recipes) {
console.log(`\n================================================================`);
console.log(`📖 Page ${recipe.page_no}: ${recipe.title} (${recipe.chapter})`);
console.log(` ID: ${recipe.id}`);
console.log(` Categories: ${recipe.categories.join(", ")}`);
console.log(` Yield: ${recipe.yield.quantity} ${recipe.yield.unit_id} (${recipe.yield.basis})`);
if (recipe.summary) console.log(` Summary: ${recipe.summary}`);
if (recipe.shelf_life) console.log(` Shelf Life: ${recipe.shelf_life.quantity} ${recipe.shelf_life.unit} (${recipe.shelf_life.storage_condition})`);
for (const comp of recipe.components) {
console.log(`\n Component: [${comp.name}] (${comp.items.length} lines):`);
for (const item of comp.items) {
const pct = item.percentage !== null ? `(${item.percentage}%)` : "";
const basis = item.basis_member ? "[BASIS]" : "";
const note = item.notes ? `[${item.notes}]` : "";
console.log(` - ${item.name.padEnd(26)} ${String(item.quantity).padStart(5)} ${item.unit_id.padEnd(5)} ${pct.padStart(9)} ${basis} ${note}`);
}
}
console.log(`\n Procedure (${recipe.steps.length} steps):`);
for (const step of recipe.steps) {
const eq = step.equipment_ids.length > 0 ? ` [Equip: ${step.equipment_ids.join(", ")}]` : "";
console.log(` ${step.order}. ${step.instruction}${eq}`);
}
if (recipe.notes.length > 0) {
console.log(`\n Chef's Notes:`);
for (const n of recipe.notes) console.log(` * ${n}`);
}
}
}
if (isSave) {
console.log(`\n💾 Ingesting ${recipes.length} recipes into Formulation database...`);
const { createMcpTools } = await import("../src/mcp/tools.ts");
const { openDatabase, refreshSiteProjection } = await import("../src/lib/database.ts");
@@ -557,8 +647,8 @@ async function main() {
for (let i = 0; i < recipes.length; i++) {
const recipe = recipes[i];
// Auto-provision lightweight ingredient records
for (const item of recipe.components[0].items) {
for (const comp of recipe.components) {
for (const item of comp.items) {
const exists = db.prepare("SELECT 1 FROM ingredients WHERE id = ?").get(item.ingredient_id);
if (!exists) {
tools.saveIngredient({
@@ -569,6 +659,7 @@ async function main() {
newIngredientsCount++;
}
}
}
const res = tools.saveRecipe(recipe);
recipeIds.push(res.recipe_id);
@@ -578,7 +669,8 @@ async function main() {
}
}
// Create or update collection: "The Pastry Chef's Little Black Book (Vol. I)"
// Update or create Master Collection
if (!targetPages) {
const collectionId = "the_pastry_chefs_little_black_book_vol_1";
tools.saveRecipeBook({
id: collectionId,
@@ -586,6 +678,7 @@ async function main() {
description: "Classic culinary pastry reference by Michael Zebrowski & Michael Mignano (477 formulations across 13 chapters).",
recipe_ids: recipeIds,
});
}
refreshSiteProjection(db);
@@ -594,7 +687,6 @@ async function main() {
console.log(`================================================================`);
console.log(` • Recipes Ingested: ${recipes.length}`);
console.log(` • New Ingredients Stubbed: ${newIngredientsCount}`);
console.log(` • Collection Created: The Pastry Chef's Little Black Book (Vol. I) (${collectionId})`);
console.log(` • Site Projection: Refreshed successfully`);
} catch (error) {
console.error("Ingestion failed:", error);
@@ -602,6 +694,7 @@ async function main() {
} finally {
db.close();
}
}
}
main().catch((err) => {