207 lines
7.3 KiB
Markdown
207 lines
7.3 KiB
Markdown
+++
|
|
categories = ["software"]
|
|
tags = ["mssql","c#"]
|
|
date = 2024-11-04T05:22:54Z
|
|
description = ""
|
|
draft = false
|
|
slug = "fooddata-central-ingest"
|
|
title = "📑 Extracting & Transforming USDA's FoodData Central Data"
|
|
author = "nicholas"
|
|
+++
|
|
|
|
USDA's *FoodData Central* is a comprehensive database of detailed information on food. Each food item has a detailed breakdown of nutrients, including macronutrients (like carbohydrates, fats, and proteins), micro-nutrients (vitamins and minerals), and other components (like water, caffeine, and alcohol). The site offers a download of the datasets in JSON format. I will write a script to transform the data to comport with the purposes of my recipe management system `Formulation`
|
|
|
|
## Steps
|
|
* Create new C# .NET Core Console Application
|
|
* .NET Core 8
|
|
* Establish connection to the database
|
|
* I create a service connection, which is just a way to let Visual Studio automatically write the connection string to my database and store it in user secrets file.
|
|
|
|
```c#
|
|
using Microsoft.Data.SqlClient;
|
|
using Microsoft.Extensions.Configuration;
|
|
|
|
var config = new ConfigurationBuilder().AddUserSecrets().Build();
|
|
var connectionString = config.GetConnectionString("FormulationDBSA");
|
|
```
|
|
|
|
* Model the data in the JSON file in C#
|
|
* I need to match Model property names with JSON property names
|
|
|
|
```c#
|
|
public class FoundationFoodData
|
|
{
|
|
[JsonPropertyName("FoundationFoods")]
|
|
public List FoundationFoods { get; set; }
|
|
}
|
|
|
|
public class FoodItem
|
|
{
|
|
[JsonPropertyName("fdcId")]
|
|
public int FdcId { get; set; }
|
|
|
|
[JsonPropertyName("description")]
|
|
public string Description { get; set; }
|
|
|
|
[JsonPropertyName("foodNutrients")]
|
|
public List FoodNutrients { get; set; }
|
|
}
|
|
|
|
public class FoodNutrient
|
|
{
|
|
|
|
[JsonPropertyName("nutrient")]
|
|
public Nutrient Nutrient { get; set; }
|
|
|
|
[JsonPropertyName("amount")]
|
|
public double Amount { get; set; }
|
|
}
|
|
|
|
public class Nutrient
|
|
{
|
|
[JsonPropertyName("number")]
|
|
public string Number { get; set; }
|
|
[JsonPropertyName("unitName")]
|
|
public string UnitName { get; set; }
|
|
}
|
|
```
|
|
|
|
* I am interested in only a few of the data model properties:
|
|
* `fdcid` - unique identifier, use this as primary key in the database
|
|
* `description` - essentially the name of the food item
|
|
* `foodNutrients` - array of nutrient objects
|
|
* I build the data model to only include those properties I am interested in:
|
|
|
|
```c#
|
|
public class Ingredient
|
|
{
|
|
public int Id { get; set; }
|
|
public string Name { get; set; }
|
|
public List Nutrients { get; set; }
|
|
}
|
|
public class IngredientNutrient
|
|
{
|
|
public string Name { get; set; }
|
|
public double Amount { get; set; }
|
|
public string Unit { get; set; }
|
|
}
|
|
```
|
|
|
|
#### Read the JSON file
|
|
```c#
|
|
var data = JsonSerializer.Deserialize(jsonData);
|
|
```
|
|
|
|
#### Create Dictionary for nutrient filtering and renaming
|
|
```c#
|
|
Dictionary NutrientFilter = new Dictionary
|
|
{
|
|
{ "Sodium", 307 },
|
|
{ "Sugar", 269 },
|
|
{ "Water", 255 },
|
|
{ "Fat", 204 },
|
|
{ "Fiber", 291 },
|
|
{ "Protein", 203 },
|
|
{ "Carbohydrate", 205 },
|
|
{ "Calories", 957 }
|
|
};
|
|
```
|
|
|
|
#### Transform the FoundationalFoods object
|
|
```c#
|
|
var ingredients = new List();
|
|
foreach (var food in data.FoundationFoods)
|
|
{
|
|
ingredients.Add(new Ingredient
|
|
{
|
|
Id = food.FdcId,
|
|
Name = food.Description,
|
|
Nutrients = food.FoodNutrients
|
|
.Where(nutrient => NutrientFilter.Any(n => n.Value.ToString() == nutrient.Nutrient.Number))
|
|
.Select(nutrient => new IngredientNutrient
|
|
{
|
|
Name = NutrientFilter.FirstOrDefault(filter => filter.Value.ToString() == nutrient.Nutrient.Number).Key,
|
|
Amount = nutrient.Amount,
|
|
Unit = nutrient.Nutrient.UnitName,
|
|
}).ToList()
|
|
});
|
|
}
|
|
```
|
|
|
|
## Results
|
|
I print formatted data to the console.
|
|
|
|
* Write the data to SQL Database
|
|
* The last step is to write the data to the database. This must wait until I sort out how to structure the database tables.
|
|
* For now, I add the fdc_id and water columns to my existing ingredient table, to see if this works.
|
|
* Once again, I use Add-Migration and Update-Database to automatically update my table schema.
|
|
|
|
```c#
|
|
foreach (var food in data.FoundationFoods)
|
|
{
|
|
ingredients.Add(new Ingredient
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
FdcId = food.FdcId,
|
|
Name = food.Description,
|
|
Sodium = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Sodium"].ToString()).Select(n => n.Amount).FirstOrDefault(),
|
|
Sugar = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Sugar"].ToString()).Select(n => n.Amount).FirstOrDefault(),
|
|
Water = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Water"].ToString()).Select(n => n.Amount).FirstOrDefault(),
|
|
Fat = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Fat"].ToString()).Select(n => n.Amount).FirstOrDefault(),
|
|
Fiber = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Fiber"].ToString()).Select(n => n.Amount).FirstOrDefault(),
|
|
Protein = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Protein"].ToString()).Select(n => n.Amount).FirstOrDefault(),
|
|
Carbohydrate = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Carbohydrate"].ToString()).Select(n => n.Amount).FirstOrDefault(),
|
|
Kilocalorie = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Calories"].ToString()).Select(n => n.Amount).FirstOrDefault(),
|
|
IsFormulation = false
|
|
});
|
|
}
|
|
|
|
DataTable ingredientData = new DataTable();
|
|
|
|
ingredientData.Columns.Add(new DataColumn("id", typeof(Guid)));
|
|
ingredientData.Columns.Add(new DataColumn("name", typeof(string)));
|
|
ingredientData.Columns.Add(new DataColumn("is_formulation", typeof(bool)));
|
|
ingredientData.Columns.Add(new DataColumn("kilocalorie", typeof(double)));
|
|
ingredientData.Columns.Add(new DataColumn("protein", typeof(double)));
|
|
ingredientData.Columns.Add(new DataColumn("fat", typeof(double)));
|
|
ingredientData.Columns.Add(new DataColumn("carbohydrate", typeof(double)));
|
|
ingredientData.Columns.Add(new DataColumn("fiber", typeof(double)));
|
|
ingredientData.Columns.Add(new DataColumn("sugar", typeof(double)));
|
|
ingredientData.Columns.Add(new DataColumn("sodium", typeof(double)));
|
|
ingredientData.Columns.Add(new DataColumn("fdc_id", typeof(int)));
|
|
ingredientData.Columns.Add(new DataColumn("water", typeof(double)));
|
|
|
|
foreach (var ingredient in ingredients)
|
|
{
|
|
ingredientData.Rows.Add(
|
|
ingredient.Id,
|
|
ingredient.Name,
|
|
ingredient.IsFormulation,
|
|
ingredient.Kilocalorie,
|
|
ingredient.Protein,
|
|
ingredient.Fat,
|
|
ingredient.Carbohydrate,
|
|
ingredient.Fiber,
|
|
ingredient.Sugar,
|
|
ingredient.Sodium,
|
|
ingredient.FdcId.HasValue ? ingredient.FdcId.Value : DBNull.Value,
|
|
ingredient.Water
|
|
);
|
|
}
|
|
|
|
using (var connection = new SqlConnection(connectionString))
|
|
{
|
|
await connection.OpenAsync();
|
|
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(connection))
|
|
{
|
|
bulkCopy.DestinationTableName = "ingredient";
|
|
await bulkCopy.WriteToServerAsync(ingredientData);
|
|
}
|
|
}
|
|
```
|
|
|
|
{{< image
|
|
src="images/json_db_ingest.exe.png"
|
|
caption="Ingest Console App" >}}
|
|
|
|
Done. |