Clean history, git LFS

2025-07-05 15:36:34 +00:00
commit 70423ce723
141 changed files with 8484 additions and 0 deletions
@@ -0,0 +1,207 @@
+++
+categories = ["software"]
+tags = ["mssql","c#"]
+date = 2024-11-04T05:22:54Z
+description = ""
+draft = false
+slug = "fooddata-central-ingest"
+title = "📑 Extracting & Transforming USDA's FoodData Central Data"
+author = "nicholas"
+++
+
+USDA's *FoodData Central* is a comprehensive database of detailed information on food. Each food item has a detailed breakdown of nutrients, including macronutrients (like carbohydrates, fats, and proteins), micro-nutrients (vitamins and minerals), and other components (like water, caffeine, and alcohol). The site offers a download of the datasets in JSON format. I will write a script to transform the data to comport with the purposes of my recipe management system `Formulation`
+
+## Steps
+ * Create new C# .NET Core Console Application
+   * .NET Core 8
+ * Establish connection to the database
+   * I create a service connection, which is just a way to let Visual Studio automatically write the connection string to my database and store it in user secrets file.
+
+```c#
+using Microsoft.Data.SqlClient;
+using Microsoft.Extensions.Configuration;
+
+var config = new ConfigurationBuilder().AddUserSecrets().Build();
+var connectionString = config.GetConnectionString("FormulationDBSA");
+```   
+
+ * Model the data in the JSON file in C#
+   * I need to match Model property names with JSON property names
+
+```c#  
+public class FoundationFoodData
+{
+    [JsonPropertyName("FoundationFoods")]
+    public List FoundationFoods { get; set; }
+}
+
+public class FoodItem
+{
+    [JsonPropertyName("fdcId")]
+    public int FdcId { get; set; }
+
+    [JsonPropertyName("description")]
+    public string Description { get; set; }
+
+    [JsonPropertyName("foodNutrients")]
+    public List FoodNutrients { get; set; }
+}
+
+public class FoodNutrient
+{
+
+    [JsonPropertyName("nutrient")]
+    public Nutrient Nutrient { get; set; }
+
+    [JsonPropertyName("amount")]
+    public double Amount { get; set; }
+}
+
+public class Nutrient
+{
+    [JsonPropertyName("number")]
+    public string Number { get; set; }
+    [JsonPropertyName("unitName")]
+    public string UnitName { get; set; }
+}
+```
+   
+ * I am interested in only a few of the data model properties:
+   * `fdcid` - unique identifier, use this as primary key in the database
+   * `description` - essentially the name of the food item
+   * `foodNutrients` - array of nutrient objects
+ * I build the data model to only include those properties I am interested in:
+  
+```c#
+    public class Ingredient
+    {
+      public int Id { get; set; }
+      public string Name { get; set; }
+      public List Nutrients { get; set; }
+    }
+    public class IngredientNutrient
+    {
+      public string Name { get; set; }
+      public double Amount { get; set; }
+      public string Unit { get; set; }
+    }
+```
+
+#### Read the JSON file
+```c#
+var data = JsonSerializer.Deserialize(jsonData);
+```
+
+#### Create Dictionary for nutrient filtering and renaming
+```c#
+Dictionary NutrientFilter = new Dictionary
+        {
+            { "Sodium", 307 },
+            { "Sugar", 269 },
+            { "Water", 255 },
+            { "Fat", 204 },
+            { "Fiber", 291 },
+            { "Protein", 203 },
+            { "Carbohydrate", 205 },
+            { "Calories", 957 }
+        };
+``` 
+
+#### Transform the FoundationalFoods object
+```c#
+var ingredients = new List();
+foreach (var food in data.FoundationFoods)
+{
+    ingredients.Add(new Ingredient
+    {
+        Id = food.FdcId,
+        Name = food.Description,
+        Nutrients = food.FoodNutrients
+        .Where(nutrient => NutrientFilter.Any(n => n.Value.ToString() == nutrient.Nutrient.Number))
+        .Select(nutrient => new IngredientNutrient
+        {
+            Name = NutrientFilter.FirstOrDefault(filter => filter.Value.ToString() == nutrient.Nutrient.Number).Key,
+            Amount = nutrient.Amount,
+            Unit = nutrient.Nutrient.UnitName,
+        }).ToList()
+    });
+}
+```
+
+## Results
+I print formatted data to the console.
+
+ * Write the data to SQL Database
+   * The last step is to write the data to the database. This must wait until I sort out how to structure the database tables.
+   * For now, I add the fdc_id and water columns to my existing ingredient table, to see if this works.
+   * Once again, I use Add-Migration and Update-Database to automatically update my table schema.
+
+```c#
+    foreach (var food in data.FoundationFoods)
+{
+    ingredients.Add(new Ingredient
+    {
+        Id = Guid.NewGuid(),
+        FdcId = food.FdcId,
+        Name = food.Description,
+        Sodium = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Sodium"].ToString()).Select(n => n.Amount).FirstOrDefault(),
+        Sugar = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Sugar"].ToString()).Select(n => n.Amount).FirstOrDefault(),
+        Water = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Water"].ToString()).Select(n => n.Amount).FirstOrDefault(),
+        Fat = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Fat"].ToString()).Select(n => n.Amount).FirstOrDefault(),
+        Fiber = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Fiber"].ToString()).Select(n => n.Amount).FirstOrDefault(),
+        Protein = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Protein"].ToString()).Select(n => n.Amount).FirstOrDefault(),
+        Carbohydrate = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Carbohydrate"].ToString()).Select(n => n.Amount).FirstOrDefault(),
+        Kilocalorie = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Calories"].ToString()).Select(n => n.Amount).FirstOrDefault(),
+        IsFormulation = false
+    });
+}
+
+DataTable ingredientData = new DataTable();
+
+ingredientData.Columns.Add(new DataColumn("id", typeof(Guid)));
+ingredientData.Columns.Add(new DataColumn("name", typeof(string)));
+ingredientData.Columns.Add(new DataColumn("is_formulation", typeof(bool)));
+ingredientData.Columns.Add(new DataColumn("kilocalorie", typeof(double)));
+ingredientData.Columns.Add(new DataColumn("protein", typeof(double)));
+ingredientData.Columns.Add(new DataColumn("fat", typeof(double)));
+ingredientData.Columns.Add(new DataColumn("carbohydrate", typeof(double)));
+ingredientData.Columns.Add(new DataColumn("fiber", typeof(double)));
+ingredientData.Columns.Add(new DataColumn("sugar", typeof(double)));
+ingredientData.Columns.Add(new DataColumn("sodium", typeof(double)));
+ingredientData.Columns.Add(new DataColumn("fdc_id", typeof(int)));
+ingredientData.Columns.Add(new DataColumn("water", typeof(double)));
+
+foreach (var ingredient in ingredients)
+{
+    ingredientData.Rows.Add(
+        ingredient.Id,
+        ingredient.Name,
+        ingredient.IsFormulation,
+        ingredient.Kilocalorie,
+        ingredient.Protein,
+        ingredient.Fat,
+        ingredient.Carbohydrate,
+        ingredient.Fiber,
+        ingredient.Sugar,
+        ingredient.Sodium,
+        ingredient.FdcId.HasValue ? ingredient.FdcId.Value : DBNull.Value,
+        ingredient.Water
+    );
+}
+
+using (var connection = new SqlConnection(connectionString))
+{
+    await connection.OpenAsync();
+    using (SqlBulkCopy bulkCopy = new SqlBulkCopy(connection))
+    {
+        bulkCopy.DestinationTableName = "ingredient";
+        await bulkCopy.WriteToServerAsync(ingredientData);
+    }
+}
+```
+
+{{< image 
+    src="images/json_db_ingest.exe.png" 
+    caption="Ingest Console App" >}}
+
+Done.