+++ categories = ["software"] tags = ["mssql","c#"] date = 2024-11-04T05:22:54Z description = "" draft = false slug = "fooddata-central-ingest" title = "📑 Extracting & Transforming USDA's FoodData Central Data" author = "nicholas" +++ USDA's *FoodData Central* is a comprehensive database of detailed information on food. Each food item has a detailed breakdown of nutrients, including macronutrients (like carbohydrates, fats, and proteins), micro-nutrients (vitamins and minerals), and other components (like water, caffeine, and alcohol). The site offers a download of the datasets in JSON format. I will write a script to transform the data to comport with the purposes of my recipe management system `Formulation` ## Steps * Create new C# .NET Core Console Application * .NET Core 8 * Establish connection to the database * I create a service connection, which is just a way to let Visual Studio automatically write the connection string to my database and store it in user secrets file. ```c# using Microsoft.Data.SqlClient; using Microsoft.Extensions.Configuration; var config = new ConfigurationBuilder().AddUserSecrets().Build(); var connectionString = config.GetConnectionString("FormulationDBSA"); ``` * Model the data in the JSON file in C# * I need to match Model property names with JSON property names ```c# public class FoundationFoodData { [JsonPropertyName("FoundationFoods")] public List FoundationFoods { get; set; } } public class FoodItem { [JsonPropertyName("fdcId")] public int FdcId { get; set; } [JsonPropertyName("description")] public string Description { get; set; } [JsonPropertyName("foodNutrients")] public List FoodNutrients { get; set; } } public class FoodNutrient { [JsonPropertyName("nutrient")] public Nutrient Nutrient { get; set; } [JsonPropertyName("amount")] public double Amount { get; set; } } public class Nutrient { [JsonPropertyName("number")] public string Number { get; set; } [JsonPropertyName("unitName")] public string UnitName { get; set; } } ``` * I am interested in only a few of the data model properties: * `fdcid` - unique identifier, use this as primary key in the database * `description` - essentially the name of the food item * `foodNutrients` - array of nutrient objects * I build the data model to only include those properties I am interested in: ```c# public class Ingredient { public int Id { get; set; } public string Name { get; set; } public List Nutrients { get; set; } } public class IngredientNutrient { public string Name { get; set; } public double Amount { get; set; } public string Unit { get; set; } } ``` #### Read the JSON file ```c# var data = JsonSerializer.Deserialize(jsonData); ``` #### Create Dictionary for nutrient filtering and renaming ```c# Dictionary NutrientFilter = new Dictionary { { "Sodium", 307 }, { "Sugar", 269 }, { "Water", 255 }, { "Fat", 204 }, { "Fiber", 291 }, { "Protein", 203 }, { "Carbohydrate", 205 }, { "Calories", 957 } }; ``` #### Transform the FoundationalFoods object ```c# var ingredients = new List(); foreach (var food in data.FoundationFoods) { ingredients.Add(new Ingredient { Id = food.FdcId, Name = food.Description, Nutrients = food.FoodNutrients .Where(nutrient => NutrientFilter.Any(n => n.Value.ToString() == nutrient.Nutrient.Number)) .Select(nutrient => new IngredientNutrient { Name = NutrientFilter.FirstOrDefault(filter => filter.Value.ToString() == nutrient.Nutrient.Number).Key, Amount = nutrient.Amount, Unit = nutrient.Nutrient.UnitName, }).ToList() }); } ``` ## Results I print formatted data to the console. * Write the data to SQL Database * The last step is to write the data to the database. This must wait until I sort out how to structure the database tables. * For now, I add the fdc_id and water columns to my existing ingredient table, to see if this works. * Once again, I use Add-Migration and Update-Database to automatically update my table schema. ```c# foreach (var food in data.FoundationFoods) { ingredients.Add(new Ingredient { Id = Guid.NewGuid(), FdcId = food.FdcId, Name = food.Description, Sodium = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Sodium"].ToString()).Select(n => n.Amount).FirstOrDefault(), Sugar = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Sugar"].ToString()).Select(n => n.Amount).FirstOrDefault(), Water = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Water"].ToString()).Select(n => n.Amount).FirstOrDefault(), Fat = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Fat"].ToString()).Select(n => n.Amount).FirstOrDefault(), Fiber = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Fiber"].ToString()).Select(n => n.Amount).FirstOrDefault(), Protein = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Protein"].ToString()).Select(n => n.Amount).FirstOrDefault(), Carbohydrate = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Carbohydrate"].ToString()).Select(n => n.Amount).FirstOrDefault(), Kilocalorie = food.FoodNutrients.Where(fn => fn.Nutrient.Number == NutrientFilter["Calories"].ToString()).Select(n => n.Amount).FirstOrDefault(), IsFormulation = false }); } DataTable ingredientData = new DataTable(); ingredientData.Columns.Add(new DataColumn("id", typeof(Guid))); ingredientData.Columns.Add(new DataColumn("name", typeof(string))); ingredientData.Columns.Add(new DataColumn("is_formulation", typeof(bool))); ingredientData.Columns.Add(new DataColumn("kilocalorie", typeof(double))); ingredientData.Columns.Add(new DataColumn("protein", typeof(double))); ingredientData.Columns.Add(new DataColumn("fat", typeof(double))); ingredientData.Columns.Add(new DataColumn("carbohydrate", typeof(double))); ingredientData.Columns.Add(new DataColumn("fiber", typeof(double))); ingredientData.Columns.Add(new DataColumn("sugar", typeof(double))); ingredientData.Columns.Add(new DataColumn("sodium", typeof(double))); ingredientData.Columns.Add(new DataColumn("fdc_id", typeof(int))); ingredientData.Columns.Add(new DataColumn("water", typeof(double))); foreach (var ingredient in ingredients) { ingredientData.Rows.Add( ingredient.Id, ingredient.Name, ingredient.IsFormulation, ingredient.Kilocalorie, ingredient.Protein, ingredient.Fat, ingredient.Carbohydrate, ingredient.Fiber, ingredient.Sugar, ingredient.Sodium, ingredient.FdcId.HasValue ? ingredient.FdcId.Value : DBNull.Value, ingredient.Water ); } using (var connection = new SqlConnection(connectionString)) { await connection.OpenAsync(); using (SqlBulkCopy bulkCopy = new SqlBulkCopy(connection)) { bulkCopy.DestinationTableName = "ingredient"; await bulkCopy.WriteToServerAsync(ingredientData); } } ``` {{< image src="images/json_db_ingest.exe.png" caption="Ingest Console App" >}} Done.