fix(ingest): support 2-page spreads and multi-component table structures in Docling parser
This commit is contained in:
+194
-101
@@ -1,9 +1,16 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Batch Docling Recipe Ingestor for "The Pastry Chef's Little Black Book, Vol. I"
|
||||
* Advanced Docling Recipe Ingestor for "The Pastry Chef's Little Black Book, Vol. I"
|
||||
*
|
||||
* Supports:
|
||||
* - 2-page facing spreads (Table on Left, Procedure on Right)
|
||||
* - Multi-component formulation splitting (Dough Packet, Butter Packet, Filling, Crust)
|
||||
* - Fractional spoon & unit fallbacks across all columns
|
||||
* - Multi-stage procedure preservation with equipment inference
|
||||
* - Shelf-life & chef's notes extraction
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/ingest-docling-book.mjs --dry-run
|
||||
* node scripts/ingest-docling-book.mjs --dry-run --pages 26-27
|
||||
* node scripts/ingest-docling-book.mjs --save
|
||||
*/
|
||||
import fs from "node:fs";
|
||||
@@ -100,12 +107,15 @@ const KNOWN_INGREDIENT_MAP = {
|
||||
"vanilla bean": "vanilla_bean",
|
||||
"vanilla beans": "vanilla_bean",
|
||||
"vanilla paste": "vanilla_extract",
|
||||
"almond extract": "almond_extract",
|
||||
"cinnamon (ground)": "cinnamon",
|
||||
"cinnamon": "cinnamon",
|
||||
"ground cinnamon": "cinnamon",
|
||||
"nutmeg": "nutmeg",
|
||||
"ground nutmeg": "nutmeg",
|
||||
"black pepper": "black_pepper",
|
||||
"white vinegar": "white_vinegar",
|
||||
"vinegar": "white_vinegar",
|
||||
"water": "water",
|
||||
"water (cold)": "water",
|
||||
"water (warm)": "water",
|
||||
@@ -139,6 +149,7 @@ const KNOWN_INGREDIENT_MAP = {
|
||||
"olive oil": "olive_oil",
|
||||
"lemon juice": "lemon_juice",
|
||||
"lemon zest": "lemon_zest",
|
||||
"lemons": "lemon",
|
||||
"orange juice": "orange_juice",
|
||||
"orange zest": "orange_zest",
|
||||
"lime juice": "lime_juice",
|
||||
@@ -211,13 +222,13 @@ function parseMetricAmount(str) {
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseUsFallback(usText, name) {
|
||||
if (!usText) {
|
||||
function parseUsFallback(text, name) {
|
||||
if (!text) {
|
||||
if (/zest/i.test(name)) return { quantity: 1, unit_id: "each", notes: "Zest of 1" };
|
||||
if (/vanilla bean/i.test(name)) return { quantity: 1, unit_id: "each", notes: "1 bean" };
|
||||
return { quantity: 1, unit_id: "gram", notes: "To taste / as needed" };
|
||||
}
|
||||
const s = usText.replace(/[\r\n\s]+/g, " ").trim().toLowerCase();
|
||||
const s = text.replace(/[\r\n\s]+/g, " ").trim().toLowerCase();
|
||||
if (s.includes("1/8") || s.includes("⅛")) return { quantity: 0.6, unit_id: "gram", notes: "⅛ tsp" };
|
||||
if (s.includes("1/4") || s.includes("¼")) return { quantity: 1.25, unit_id: "gram", notes: "¼ tsp" };
|
||||
if (s.includes("1/2") || s.includes("½")) return { quantity: 2.5, unit_id: "gram", notes: "½ tsp" };
|
||||
@@ -233,10 +244,10 @@ function parseUsFallback(usText, name) {
|
||||
|
||||
const ozMatch = s.match(/^([\d.]+)\s*oz$/);
|
||||
if (ozMatch && parseFloat(ozMatch[1]) > 0) {
|
||||
return { quantity: Math.round(parseFloat(ozMatch[1]) * 28.3495 * 100) / 100, unit_id: "gram", notes: usText };
|
||||
return { quantity: Math.round(parseFloat(ozMatch[1]) * 28.3495 * 100) / 100, unit_id: "gram", notes: text };
|
||||
}
|
||||
|
||||
return { quantity: 1, unit_id: "gram", notes: usText };
|
||||
return { quantity: 1, unit_id: "gram", notes: text };
|
||||
}
|
||||
|
||||
function cleanIngredientName(raw) {
|
||||
@@ -361,6 +372,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
|
||||
const isIngredientTable = headerRow && [...headerRow.values()].some((v) => /ingredients/i.test(v));
|
||||
if (!isIngredientTable) continue;
|
||||
|
||||
// Find Recipe Title on this page
|
||||
const titleNode = page.texts.find(
|
||||
(t) => t.label === "section_header" && !/procedure|chef's notes|notes|table of contents|scaling|baking/i.test(t.text)
|
||||
);
|
||||
@@ -374,7 +386,11 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
|
||||
usedSlugs.set(slugId, 1);
|
||||
}
|
||||
|
||||
const items = [];
|
||||
// Parse ingredients into components
|
||||
const components = [];
|
||||
let currentComponent = { id: "main", name: "Main", items: [] };
|
||||
components.push(currentComponent);
|
||||
|
||||
let totalYieldGrams = null;
|
||||
|
||||
for (let r = 1; r <= maxRow; r++) {
|
||||
@@ -386,24 +402,37 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
|
||||
const usText = row.get(2) || "";
|
||||
|
||||
if (/total weight/i.test(ingText)) {
|
||||
const parsedTotal = parseMetricAmount(metricText);
|
||||
const parsedTotal = parseMetricAmount(metricText) || parseMetricAmount(usText);
|
||||
if (parsedTotal) totalYieldGrams = parsedTotal.quantity;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!ingText) continue;
|
||||
if (ingText.endsWith(":") && !metricText && !usText) continue;
|
||||
|
||||
// Detect sub-component headers inside tables like "Dough Packet (Détrempe):" or "Filling:"
|
||||
if (ingText.endsWith(":") && !metricText && !usText) {
|
||||
const compName = ingText.replace(/:$/, "").trim();
|
||||
const compSlug = slugify(compName);
|
||||
if (currentComponent.items.length === 0 && components.length === 1) {
|
||||
currentComponent.id = compSlug;
|
||||
currentComponent.name = compName;
|
||||
} else {
|
||||
currentComponent = { id: compSlug, name: compName, items: [] };
|
||||
components.push(currentComponent);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const { name, notes: parenNotes } = cleanIngredientName(ingText);
|
||||
const ingredientId = inferIngredientId(name);
|
||||
const parsedMetric = parseMetricAmount(metricText);
|
||||
|
||||
let parsedMetric = parseMetricAmount(metricText);
|
||||
let notes = parenNotes;
|
||||
let quantity = parsedMetric ? parsedMetric.quantity : 0;
|
||||
let unitId = parsedMetric ? parsedMetric.unit_id : "gram";
|
||||
|
||||
if (quantity <= 0) {
|
||||
const fallback = parseUsFallback(usText, name);
|
||||
const fallback = parseUsFallback(metricText || usText, name);
|
||||
quantity = fallback.quantity;
|
||||
unitId = fallback.unit_id;
|
||||
if (fallback.notes) {
|
||||
@@ -415,7 +444,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
|
||||
}
|
||||
}
|
||||
|
||||
items.push({
|
||||
currentComponent.items.push({
|
||||
raw_name: ingText,
|
||||
clean_name: name,
|
||||
ingredient_id: ingredientId,
|
||||
@@ -427,40 +456,61 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
|
||||
});
|
||||
}
|
||||
|
||||
if (items.length === 0) continue;
|
||||
// Filter out empty components
|
||||
const validComponents = components.filter((c) => c.items.length > 0);
|
||||
if (validComponents.length === 0) continue;
|
||||
|
||||
const flourBasisWeight = items
|
||||
// Calculate Baker's Percentages across all components
|
||||
const allItems = validComponents.flatMap((c) => c.items);
|
||||
const flourBasisWeight = allItems
|
||||
.filter((i) => i.basis_member)
|
||||
.reduce((sum, i) => sum + i.quantity, 0);
|
||||
|
||||
const computedItems = items.map((item, idx) => {
|
||||
let pct = null;
|
||||
if (flourBasisWeight > 0 && item.quantity > 0) {
|
||||
pct = Number(((item.quantity / flourBasisWeight) * 100).toFixed(2));
|
||||
}
|
||||
return {
|
||||
id: `line_${String(idx + 1).padStart(2, "0")}_${item.ingredient_id}`,
|
||||
ingredient_id: item.ingredient_id,
|
||||
name: titleCase(item.clean_name),
|
||||
quantity: item.quantity,
|
||||
unit_id: item.unit_id,
|
||||
percentage: pct,
|
||||
basis_member: item.basis_member,
|
||||
notes: item.notes,
|
||||
};
|
||||
});
|
||||
let itemCounter = 1;
|
||||
const formattedComponents = validComponents.map((comp) => ({
|
||||
id: comp.id,
|
||||
name: comp.name,
|
||||
notes: [],
|
||||
items: comp.items.map((item) => {
|
||||
let pct = null;
|
||||
if (flourBasisWeight > 0 && item.quantity > 0) {
|
||||
pct = Number(((item.quantity / flourBasisWeight) * 100).toFixed(2));
|
||||
}
|
||||
return {
|
||||
id: `line_${String(itemCounter++).padStart(2, "0")}_${item.ingredient_id}`,
|
||||
ingredient_id: item.ingredient_id,
|
||||
name: titleCase(item.clean_name),
|
||||
quantity: item.quantity,
|
||||
unit_id: item.unit_id,
|
||||
percentage: pct,
|
||||
basis_member: item.basis_member,
|
||||
notes: item.notes,
|
||||
};
|
||||
}),
|
||||
}));
|
||||
|
||||
// Find procedure steps on current page or facing spread page (pageNo + 1)
|
||||
const textSources = [...page.texts];
|
||||
const nextPage = pagesMap.get(pageNo + 1);
|
||||
if (nextPage && nextPage.tables.length === 0) {
|
||||
textSources.push(...nextPage.texts);
|
||||
}
|
||||
|
||||
const steps = [];
|
||||
let currentSectionPrefix = "";
|
||||
let inProcedure = false;
|
||||
let inChefNotes = false;
|
||||
const chefNotesList = [];
|
||||
const narrativeTexts = [];
|
||||
|
||||
for (const textNode of page.texts) {
|
||||
for (const textNode of textSources) {
|
||||
const text = textNode.text.trim();
|
||||
if (/^procedure:?$/i.test(text)) {
|
||||
|
||||
// Match procedure section headers like "Dough Packet Procedure:", "Assembly Procedure:", "Procedure:"
|
||||
if (/procedure:?$/i.test(text)) {
|
||||
inProcedure = true;
|
||||
inChefNotes = false;
|
||||
currentSectionPrefix = text.replace(/procedure:?$/i, "").trim();
|
||||
continue;
|
||||
}
|
||||
if (/^chef's notes:?$/i.test(text)) {
|
||||
@@ -471,12 +521,15 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
|
||||
|
||||
if (inProcedure) {
|
||||
if (textNode.label === "list_item" || textNode.label === "text") {
|
||||
steps.push({
|
||||
id: `step_${steps.length + 1}`,
|
||||
order: steps.length + 1,
|
||||
instruction: text,
|
||||
equipment_ids: inferEquipment(text),
|
||||
});
|
||||
if (!/^\d+$/.test(text)) {
|
||||
const prefix = currentSectionPrefix ? `[${currentSectionPrefix}] ` : "";
|
||||
steps.push({
|
||||
id: `step_${steps.length + 1}`,
|
||||
order: steps.length + 1,
|
||||
instruction: `${prefix}${text}`,
|
||||
equipment_ids: inferEquipment(text),
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if (inChefNotes) {
|
||||
if (textNode.label === "list_item") {
|
||||
@@ -487,7 +540,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
|
||||
}
|
||||
}
|
||||
|
||||
const sumWeight = computedItems.reduce((s, i) => s + (i.unit_id === "gram" ? i.quantity : 0), 0);
|
||||
const sumWeight = allItems.reduce((s, i) => s + (i.unit_id === "gram" ? i.quantity : 0), 0);
|
||||
const yieldQuantity = totalYieldGrams || (sumWeight > 0 ? sumWeight : 1000);
|
||||
|
||||
recipes.push({
|
||||
@@ -507,13 +560,7 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
|
||||
unit_id: "gram",
|
||||
basis: "theoretical",
|
||||
},
|
||||
components: [
|
||||
{
|
||||
id: "main",
|
||||
name: "Main",
|
||||
items: computedItems,
|
||||
},
|
||||
],
|
||||
components: formattedComponents,
|
||||
steps: steps.length > 0 ? steps : [{ id: "step_1", order: 1, instruction: "Prepare formulation according to standard pastry method.", equipment_ids: [] }],
|
||||
notes: chefNotesList,
|
||||
shelf_life: parseShelfLife(chefNotesList),
|
||||
@@ -530,77 +577,123 @@ export function parseDoclingBook(doclingJson, targetPages = null) {
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const isSave = args.includes("--save");
|
||||
const isDryRun = args.includes("--dry-run");
|
||||
|
||||
let targetPages = null;
|
||||
const pageIdx = args.indexOf("--page");
|
||||
if (pageIdx !== -1 && args[pageIdx + 1]) {
|
||||
targetPages = [parseInt(args[pageIdx + 1], 10)];
|
||||
}
|
||||
const pagesIdx = args.indexOf("--pages");
|
||||
if (pagesIdx !== -1 && args[pagesIdx + 1]) {
|
||||
const [start, end] = args[pagesIdx + 1].split("-").map((n) => parseInt(n, 10));
|
||||
targetPages = [];
|
||||
for (let p = start; p <= end; p++) targetPages.push(p);
|
||||
}
|
||||
|
||||
console.log("Loading Docling JSON from file.json...");
|
||||
const rawData = fs.readFileSync(doclingPath, "utf8");
|
||||
const doc = JSON.parse(rawData);
|
||||
console.log(`Document loaded: ${doc.texts?.length || 0} texts, ${doc.tables?.length || 0} tables.`);
|
||||
|
||||
const recipes = parseDoclingBook(doc);
|
||||
console.log(`\nFound ${recipes.length} formulation(s) across 13 chapters.`);
|
||||
const recipes = parseDoclingBook(doc, targetPages);
|
||||
console.log(`\nFound ${recipes.length} formulation(s).`);
|
||||
|
||||
if (!isSave) {
|
||||
console.log("Run with '--save' to commit to database.");
|
||||
return;
|
||||
}
|
||||
// Detailed inspect for targeted page runs
|
||||
if (targetPages && targetPages.length <= 5) {
|
||||
for (const recipe of recipes) {
|
||||
console.log(`\n================================================================`);
|
||||
console.log(`📖 Page ${recipe.page_no}: ${recipe.title} (${recipe.chapter})`);
|
||||
console.log(` ID: ${recipe.id}`);
|
||||
console.log(` Categories: ${recipe.categories.join(", ")}`);
|
||||
console.log(` Yield: ${recipe.yield.quantity} ${recipe.yield.unit_id} (${recipe.yield.basis})`);
|
||||
if (recipe.summary) console.log(` Summary: ${recipe.summary}`);
|
||||
if (recipe.shelf_life) console.log(` Shelf Life: ${recipe.shelf_life.quantity} ${recipe.shelf_life.unit} (${recipe.shelf_life.storage_condition})`);
|
||||
|
||||
console.log(`\n💾 Ingesting ${recipes.length} recipes into Formulation database...`);
|
||||
const { createMcpTools } = await import("../src/mcp/tools.ts");
|
||||
const { openDatabase, refreshSiteProjection } = await import("../src/lib/database.ts");
|
||||
const db = openDatabase({ readOnly: false });
|
||||
const tools = createMcpTools(() => db);
|
||||
|
||||
try {
|
||||
let newIngredientsCount = 0;
|
||||
const recipeIds = [];
|
||||
|
||||
for (let i = 0; i < recipes.length; i++) {
|
||||
const recipe = recipes[i];
|
||||
|
||||
// Auto-provision lightweight ingredient records
|
||||
for (const item of recipe.components[0].items) {
|
||||
const exists = db.prepare("SELECT 1 FROM ingredients WHERE id = ?").get(item.ingredient_id);
|
||||
if (!exists) {
|
||||
tools.saveIngredient({
|
||||
id: item.ingredient_id,
|
||||
name: item.name,
|
||||
categories: ["pantry", "baking", "imported_stub"],
|
||||
});
|
||||
newIngredientsCount++;
|
||||
for (const comp of recipe.components) {
|
||||
console.log(`\n Component: [${comp.name}] (${comp.items.length} lines):`);
|
||||
for (const item of comp.items) {
|
||||
const pct = item.percentage !== null ? `(${item.percentage}%)` : "";
|
||||
const basis = item.basis_member ? "[BASIS]" : "";
|
||||
const note = item.notes ? `[${item.notes}]` : "";
|
||||
console.log(` - ${item.name.padEnd(26)} ${String(item.quantity).padStart(5)} ${item.unit_id.padEnd(5)} ${pct.padStart(9)} ${basis} ${note}`);
|
||||
}
|
||||
}
|
||||
|
||||
const res = tools.saveRecipe(recipe);
|
||||
recipeIds.push(res.recipe_id);
|
||||
console.log(`\n Procedure (${recipe.steps.length} steps):`);
|
||||
for (const step of recipe.steps) {
|
||||
const eq = step.equipment_ids.length > 0 ? ` [Equip: ${step.equipment_ids.join(", ")}]` : "";
|
||||
console.log(` ${step.order}. ${step.instruction}${eq}`);
|
||||
}
|
||||
|
||||
if ((i + 1) % 50 === 0 || i + 1 === recipes.length) {
|
||||
console.log(` [${i + 1}/${recipes.length}] Processed: ${recipe.title} (${res.recipe_id})`);
|
||||
if (recipe.notes.length > 0) {
|
||||
console.log(`\n Chef's Notes:`);
|
||||
for (const n of recipe.notes) console.log(` * ${n}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create or update collection: "The Pastry Chef's Little Black Book (Vol. I)"
|
||||
const collectionId = "the_pastry_chefs_little_black_book_vol_1";
|
||||
tools.saveRecipeBook({
|
||||
id: collectionId,
|
||||
name: "The Pastry Chef's Little Black Book (Vol. I)",
|
||||
description: "Classic culinary pastry reference by Michael Zebrowski & Michael Mignano (477 formulations across 13 chapters).",
|
||||
recipe_ids: recipeIds,
|
||||
});
|
||||
if (isSave) {
|
||||
console.log(`\n💾 Ingesting ${recipes.length} recipes into Formulation database...`);
|
||||
const { createMcpTools } = await import("../src/mcp/tools.ts");
|
||||
const { openDatabase, refreshSiteProjection } = await import("../src/lib/database.ts");
|
||||
const db = openDatabase({ readOnly: false });
|
||||
const tools = createMcpTools(() => db);
|
||||
|
||||
refreshSiteProjection(db);
|
||||
try {
|
||||
let newIngredientsCount = 0;
|
||||
const recipeIds = [];
|
||||
|
||||
console.log(`\n================================================================`);
|
||||
console.log(`🎉 INGESTION COMPLETE!`);
|
||||
console.log(`================================================================`);
|
||||
console.log(` • Recipes Ingested: ${recipes.length}`);
|
||||
console.log(` • New Ingredients Stubbed: ${newIngredientsCount}`);
|
||||
console.log(` • Collection Created: The Pastry Chef's Little Black Book (Vol. I) (${collectionId})`);
|
||||
console.log(` • Site Projection: Refreshed successfully`);
|
||||
} catch (error) {
|
||||
console.error("Ingestion failed:", error);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
db.close();
|
||||
for (let i = 0; i < recipes.length; i++) {
|
||||
const recipe = recipes[i];
|
||||
|
||||
for (const comp of recipe.components) {
|
||||
for (const item of comp.items) {
|
||||
const exists = db.prepare("SELECT 1 FROM ingredients WHERE id = ?").get(item.ingredient_id);
|
||||
if (!exists) {
|
||||
tools.saveIngredient({
|
||||
id: item.ingredient_id,
|
||||
name: item.name,
|
||||
categories: ["pantry", "baking", "imported_stub"],
|
||||
});
|
||||
newIngredientsCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const res = tools.saveRecipe(recipe);
|
||||
recipeIds.push(res.recipe_id);
|
||||
|
||||
if ((i + 1) % 50 === 0 || i + 1 === recipes.length) {
|
||||
console.log(` [${i + 1}/${recipes.length}] Processed: ${recipe.title} (${res.recipe_id})`);
|
||||
}
|
||||
}
|
||||
|
||||
// Update or create Master Collection
|
||||
if (!targetPages) {
|
||||
const collectionId = "the_pastry_chefs_little_black_book_vol_1";
|
||||
tools.saveRecipeBook({
|
||||
id: collectionId,
|
||||
name: "The Pastry Chef's Little Black Book (Vol. I)",
|
||||
description: "Classic culinary pastry reference by Michael Zebrowski & Michael Mignano (477 formulations across 13 chapters).",
|
||||
recipe_ids: recipeIds,
|
||||
});
|
||||
}
|
||||
|
||||
refreshSiteProjection(db);
|
||||
|
||||
console.log(`\n================================================================`);
|
||||
console.log(`🎉 INGESTION COMPLETE!`);
|
||||
console.log(`================================================================`);
|
||||
console.log(` • Recipes Ingested: ${recipes.length}`);
|
||||
console.log(` • New Ingredients Stubbed: ${newIngredientsCount}`);
|
||||
console.log(` • Site Projection: Refreshed successfully`);
|
||||
} catch (error) {
|
||||
console.error("Ingestion failed:", error);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user